starred/nix-config
1
0
Fork 0
mirror of https://github.com/ryan4yin/nix-config.git synced 2026-01-11 20:40:24 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat: add configs for kubevirt-lab-1

Browse Source
This commit is contained in:
Ryan Yin
2024-03-28 22:22:32 +08:00
parent 8d845f6fd0
commit 69fafe0c7c
10 changed files with 260 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
hosts/k8s/disko-config/README.md
View File

@@ -4,15 +4,16 @@ Generate LUKS keyfile to encrypt the root partition, it's used by disko.
```bash
# partition the usb stick
parted /dev/sdb -- mklabel gpt
parted /dev/sdb -- mkpart primary 2M 512MB
parted /dev/sdb -- mkpart primary 512MB 1024MB
mkfs.fat -F 32 -n NIXOS_DSC /dev/sdb1
mkfs.fat -F 32 -n NIXOS_K3S /dev/sdb2
DEV=/dev/sdX
parted $DEV -- mklabel gpt
parted $DEV -- mkpart primary 2M 512MB
parted $DEV -- mkpart primary 512MB 1024MB
mkfs.fat -F 32 -n NIXOS_DSC ${DEV}1
mkfs.fat -F 32 -n NIXOS_K3S ${DEV}2
# Generate a keyfile from the true random number generator
KEYFILE=./kubevirt-luks-keyfile
dd bs=8192 count=4 iflag=fullblock if=/dev/random of=$KEYFILE
dd bs=512 count=64 iflag=fullblock if=/dev/random of=$KEYFILE
# generate token for k3s
K3S_TOKEN_FILE=./kubevirt-k3s-token
@@ -20,12 +21,25 @@ K3S_TOKEN=$(grep -ao '[A-Za-z0-9]' < /dev/random | head -64 | tr -d '\n' ; echo
echo $K3S_TOKEN > $K3S_TOKEN_FILE
# copy the keyfile and token to the usb stick
KEYFILE=./kubevirt-luks-keyfile
DEVICE=/dev/disk/by-label/NIXOS_DSC
dd bs=8192 count=4 iflag=fullblock if=$KEYFILE of=$DEVICE
# seek=128 skip N obs-sized output blocks to avoid overwriting the filesystem header
dd bs=512 count=64 iflag=fullblock seek=128 if=$KEYFILE of=$DEVICE
K3S_TOKEN_FILE=./kubevirt-k3s-token
USB_PATH=/run/media/ryan/NIXOS_K3S
cp $K3S_TOKEN_FILE $USB_PATH
```
### 2. Partition the SSD & install NixOS via disko
```bash
# enter an shell with git/vim/ssh-agent/gnumake available
nix-shell -p git vim gnumake
# clone this repository
git clone https://github.com/ryan4yin/nix-config.git
cd nix-config
sudo nix run --experimental-features "nix-command flakes" 'github:nix-community/disko#disko-install' -- \
--write-efi-boot-entries --flake .#kubevirt-shoryu --disk main /dev/nvme0n1
```

169
hosts/k8s/disko-config/kubevirt-disko-fs.nix
View File

@@ -1,4 +1,7 @@
{
# required by impermanence
fileSystems."/persistent".neededForBoot = true;
# contains the k3s's token
fileSystems."/run/media/nixos_k3s" = {
device = "/dev/disk/by-label/NIXOS_K3S";
@@ -7,90 +10,92 @@
};
disko.devices = {
disk = {
sda = {
type = "disk";
device = "/dev/nvme0n1";
content = {
type = "gpt";
partitions = {
# The EFI & Boot partition
ESP = {
size = "630M";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot/efi";
mountOptions = [
"defaults"
];
};
nodev."/" = {
fsType = "tmpfs";
mountOptions = [
"size=4G"
"defaults"
# set mode to 755, otherwise systemd will set it to 777, which cause problems.
# relatime: Update inode access times relative to modify or change time.
"mode=755"
];
};
disk.main = {
type = "disk";
# When using disko-install, we will overwrite this value from the commandline
device = "/dev/nvme0n1"; # The device to partition
content = {
type = "gpt";
partitions = {
# The EFI & Boot partition
ESP = {
size = "630M";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [
"defaults"
];
};
# The root partition
luks = {
size = "100%";
};
# The root partition
luks = {
size = "100%";
content = {
type = "luks";
name = "encrypted";
settings = {
keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
# The maximum size of the keyfile is 8192 bytes
keyFileSize = 512 * 64; # match the `bs * count` of the `dd` command
keyFileOffset = 512 * 128; # match the `bs * skip` of the `dd` command
fallbackToPassword = true;
allowDiscards = true;
};
# Whether to add a boot.initrd.luks.devices entry for the specified disk.
initrdUnlock = true;
# encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
# cryptsetup luksFormat
extraFormatArgs = [
"--type luks2"
"--cipher aes-xts-plain64"
"--hash sha512"
"--iter-time 5000"
"--key-size 256"
"--pbkdf argon2id"
# use true random data from /dev/random, will block until enough entropy is available
"--use-random"
];
extraOpenArgs = [
"--timeout 10"
];
content = {
type = "luks";
name = "crypted";
settings = {
keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
keyFileSize = 8192 * 4; # The maxium size of the keyfile is 8192 bytes
keyFileOffset = 0;
fallbackToPassword = true;
allowDiscards = true;
};
# Whether to add a boot.initrd.luks.devices entry for the specified disk.
initrdUnlock = true;
# encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
# cryptsetup luksFormat
extraFormatArgs = [
"--type luks2"
"--cipher aes-xts-plain64"
"--hash sha512"
"--iter-time 5000"
"--key-size 256"
"--pbkdf argon2id"
# use true random data from /dev/random, will block until enough entropy is available
"--use-random"
];
extraOpenArgs = [
"--timeout 10"
];
content = {
type = "btrfs";
extraArgs = ["-f"];
subvolumes = {
"@root" = {
mountpoint = "/";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@home" = {
mountpoint = "/home";
mountOptions = ["compress-force=zstd:1"];
};
"@lib" = {
mountpoint = "/var/lib";
mountOptions = ["compress-force=zstd:1"];
};
"@nix" = {
mountpoint = "/nix";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@tmp" = {
mountpoint = "/tmp";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@snapshots" = {
mountpoint = "/snapshots";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@swap" = {
mountpoint = "/swap";
swap.swapfile.size = "8192M";
};
type = "btrfs";
extraArgs = ["-f"];
subvolumes = {
"@nix" = {
mountpoint = "/nix";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@persistent" = {
mountpoint = "/persistent";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@tmp" = {
mountpoint = "/tmp";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@snapshots" = {
mountpoint = "/snapshots";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@swap" = {
mountpoint = "/swap";
swap.swapfile.size = "16384M";
};
};
};

6
hosts/k8s/kubevirt-shoryu/default.nix
View File

@@ -8,7 +8,7 @@
}: let
# MoreFine - S500Plus
hostName = "kubevirt-shoryu"; # Define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
inherit (myvars) networking;
@@ -16,7 +16,7 @@
k3sModule = mylib.genK3sServerModule {
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
# the first node in the cluster should be the one to initialize the cluster
clusterInit = true;
};
@@ -26,6 +26,8 @@ in {
++ [
disko.nixosModules.default
../disko-config/kubevirt-disko-fs.nix
./hardware-configuration.nix
./impermanence.nix
coreModule
k3sModule
];

49
hosts/k8s/kubevirt-shoryu/hardware-configuration.nix Normal file
View File

@@ -0,0 +1,49 @@
{
config,
lib,
pkgs,
modulesPath,
...
}: {
imports = [
(modulesPath + "/installer/scan/not-detected.nix")
];
boot.loader = {
# depending on how you configured your disk mounts, change this to /boot or /boot/efi.
efi.efiSysMountPoint = "/boot/";
efi.canTouchEfiVariables = true;
systemd-boot.enable = true;
};
# clear /tmp on boot to get a stateless /tmp directory.
boot.tmp.cleanOnBoot = true;
boot.initrd.availableKernelModules = ["xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod"];
boot.kernelModules = ["kvm-amd" "vfio-pci"];
boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu
# Enable binfmt emulation of aarch64-linux, this is required for cross compilation.
boot.binfmt.emulatedSystems = ["aarch64-linux" "riscv64-linux"];
# supported file systems, so we can mount any removable disks with these filesystems
boot.supportedFilesystems = [
"ext4"
"btrfs"
"xfs"
"ntfs"
"fat"
"vfat"
"cifs" # mount windows share
];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp5s0.useDHCP = lib.mkDefault true;
# networking.interfaces.wlo1.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

53
hosts/k8s/kubevirt-shoryu/impermanence.nix Normal file
View File

@@ -0,0 +1,53 @@
{
impermanence,
pkgs,
...
}: {
imports = [
impermanence.nixosModules.impermanence
];
environment.systemPackages = [
# `sudo ncdu -x /`
pkgs.ncdu
];
# NOTE: impermanence only mounts the directory/file list below to /persistent
# If the directory/file already exists in the root filesystem, you should
# move those files/directories to /persistent first!
environment.persistence."/persistent" = {
# sets the mount option x-gvfs-hide on all the bind mounts
# to hide them from the file manager
hideMounts = true;
directories = [
"/etc/NetworkManager/system-connections"
"/etc/ssh"
"/etc/nix/inputs"
"/etc/secureboot" # lanzaboote - secure boot
# my secrets
"/etc/agenix/"
"/var/log"
"/var/lib"
# k3s related
"/etc/iscsi"
"/etc/rancher"
];
files = [
"/etc/machine-id"
];
# the following directories will be passed to /persistent/home/$USER
users.ryan = {
directories = [
"codes"
"nix-config"
"tmp"
];
files = [
".config/nushell/history.txt"
];
};
};
}

4
hosts/k8s/kubevirt-shushou/default.nix
View File

@@ -16,7 +16,7 @@
k3sModule = mylib.genK3sServerModule {
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
};
in {
@@ -25,6 +25,8 @@ in {
++ [
disko.nixosModules.default
../disko-config/kubevirt-disko-fs.nix
../kubevirt-shoryu/hardware-configuration.nix
../kubevirt-shoryu/impermanence.nix
coreModule
k3sModule
];

4
hosts/k8s/kubevirt-youko/default.nix
View File

@@ -16,7 +16,7 @@
k3sModule = mylib.genK3sServerModule {
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
};
in {
@@ -25,6 +25,8 @@ in {
++ [
disko.nixosModules.default
../disko-config/kubevirt-disko-fs.nix
../kubevirt-shoryu/hardware-configuration.nix
../kubevirt-shoryu/impermanence.nix
coreModule
k3sModule
];

10
lib/genK3sAgentModule.nix
View File

@@ -14,8 +14,12 @@ in {
role = "agent";
serverAddr = "https://${serverIp}:6443";
# https://docs.k3s.io/cli/agent
extraFlags =
" --node-label=node-type=worker"
+ " --data-dir /var/lib/rancher/k3s";
extraFlags = let
flagList = [
"--node-label=node-type=worker"
"--data-dir /var/lib/rancher/k3s"
];
in
pkgs.lib.concatStringsSep " " flagList;
};
}

46
lib/genK3sServerModule.nix
View File

@@ -8,6 +8,7 @@
# and other servers must connect to it using `serverAddr`.
serverIp ? null,
clusterInit ? (serverIp == null),
addTaints ? false,
...
}: let
package = pkgs.k3s_1_29;
@@ -18,6 +19,8 @@ in {
kubectl
istioctl
kubernetes-helm
cilium-cli
fluxcd
skopeo
dive # explore docker layers
@@ -33,20 +36,33 @@ in {
role = "server";
# https://docs.k3s.io/cli/server
extraFlags =
" --write-kubeconfig ${kubeconfigFile}"
+ " --write-kubeconfig-mode 644"
+ " --service-node-port-range 80-32767"
+ " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+ " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master
+ " --data-dir /var/lib/rancher/k3s"
+ " --etcd-expose-metrics true"
+ " --etcd-snapshot-schedule-cron '0 */12 * * *'"
# disable some features we don't need
+ " --disable-helm-controller" # we use fluxcd instead
+ " --disable=traefik" # deploy our own ingress controller instead
+ " --disable=servicelb" # we use kube-vip instead
+ " --flannel-backend=none" # we use cilium instead
+ " --disable-network-policy";
extraFlags = let
flagList =
[
"--write-kubeconfig ${kubeconfigFile}"
"--write-kubeconfig-mode=644"
"--service-node-port-range=80-32767"
"--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
"--data-dir /var/lib/rancher/k3s"
"--etcd-expose-metrics=true"
"--etcd-snapshot-schedule-cron='0 */12 * * *'"
# disable some features we don't need
"--disable-helm-controller" # we use fluxcd instead
"--disable=traefik" # deploy our own ingress controller instead
"--disable=servicelb" # we use kube-vip instead
"--flannel-backend=none" # we use cilium instead
"--disable-network-policy"
]
# prevent workloads from running on the master
++ (pkgs.lib.optionals addTaints ["--node-taint=CriticalAddonsOnly=true:NoExecute"]);
in
pkgs.lib.concatStringsSep " " flagList;
};
# create symlinks to link k3s's cni directory to the one used by almost all CNI plugins
systemd.tmpfiles.rules = [
"L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/current/bin"
# seems like k3s's containerd will create /etc/cni/net.d, so we don't need to create a symlink for it
# "L+ /etc/cni/net.d - - - - /var/lib/rancher/k3s/agent/etc/cni/net.d"
];
}

1
modules/nixos/server/server.nix
View File

@@ -3,4 +3,5 @@
../base
../../base.nix
];
boot.loader.timeout = 3; # wait for 3 seconds to select the boot entry
}