feat: IPv6 (#192)

* feat: custom ipv6 routes for all hosts
* fix: ipv6 - k3s+cilium

hosts/idols-ai/default.nix

@@ -33,7 +33,6 @@ in {
   networking.useNetworkd = true;
   systemd.network.enable = true;
 
-  # Add ipv4 address to the bridge.
   systemd.network.networks."10-${iface}" = {
     matchConfig.Name = [iface];
     networkConfig = {

hosts/idols-aquamarine/default.nix

@@ -13,6 +13,10 @@
 #############################################################
 let
   hostName = "aquamarine"; # Define your hostname.
+
+  inherit (myvars.networking) defaultGateway defaultGateway6 nameservers;
+  inherit (myvars.networking.hostsAddr.${hostName}) iface ipv4;
+  ipv4WithMask = "${ipv4}/24";
 in {
   imports =
     (mylib.scanPaths ./.)
@@ -43,9 +47,36 @@ in {
 
   networking = {
     inherit hostName;
-    inherit (myvars.networking) defaultGateway nameservers;
+
-    inherit (myvars.networking.hostsInterface.${hostName}) interfaces;
+    # we use networkd instead
     networkmanager.enable = false;
+    useDHCP = false;
+  };
+
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
+  systemd.network.networks."10-${iface}" = {
+    matchConfig.Name = [iface];
+    networkConfig = {
+      Address = [ipv4WithMask];
+      DNS = nameservers;
+      DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
+      IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC)
+      LinkLocalAddressing = "ipv6";
+    };
+    routes = [
+      {
+        Destination = "0.0.0.0/0";
+        Gateway = defaultGateway;
+      }
+      {
+        Destination = "::/0";
+        Gateway = defaultGateway6;
+        GatewayOnLink = true; # it's a gateway on local link.
+      }
+    ];
+    linkConfig.RequiredForOnline = "routable";
   };
 
   # This value determines the NixOS release from which the default

hosts/idols-kana/default.nix

@@ -10,6 +10,10 @@
 #############################################################
 let
   hostName = "kana"; # Define your hostname.
+
+  inherit (myvars.networking) defaultGateway defaultGateway6 nameservers;
+  inherit (myvars.networking.hostsAddr.${hostName}) iface ipv4;
+  ipv4WithMask = "${ipv4}/24";
 in {
   imports = mylib.scanPaths ./.;
 
@@ -30,9 +34,36 @@ in {
 
   networking = {
     inherit hostName;
-    inherit (myvars.networking) defaultGateway nameservers;
+
-    inherit (myvars.networking.hostsInterface.${hostName}) interfaces;
+    # we use networkd instead
     networkmanager.enable = false;
+    useDHCP = false;
+  };
+
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
+  systemd.network.networks."10-${iface}" = {
+    matchConfig.Name = [iface];
+    networkConfig = {
+      Address = [ipv4WithMask];
+      DNS = nameservers;
+      DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
+      IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC)
+      LinkLocalAddressing = "ipv6";
+    };
+    routes = [
+      {
+        Destination = "0.0.0.0/0";
+        Gateway = defaultGateway;
+      }
+      {
+        Destination = "::/0";
+        Gateway = defaultGateway6;
+        GatewayOnLink = true; # it's a gateway on local link.
+      }
+    ];
+    linkConfig.RequiredForOnline = "routable";
   };
 
   # This value determines the NixOS release from which the default

hosts/idols-ruby/default.nix

@@ -10,6 +10,10 @@
 #############################################################
 let
   hostName = "ruby"; # Define your hostname.
+
+  inherit (myvars.networking) defaultGateway defaultGateway6 nameservers;
+  inherit (myvars.networking.hostsAddr.${hostName}) iface ipv4;
+  ipv4WithMask = "${ipv4}/24";
 in {
   imports = mylib.scanPaths ./.;
 
@@ -32,9 +36,36 @@ in {
 
   networking = {
     inherit hostName;
-    inherit (myvars.networking) defaultGateway nameservers;
+
-    inherit (myvars.networking.hostsInterface.${hostName}) interfaces;
+    # we use networkd instead
     networkmanager.enable = false;
+    useDHCP = false;
+  };
+
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
+  systemd.network.networks."10-${iface}" = {
+    matchConfig.Name = [iface];
+    networkConfig = {
+      Address = [ipv4WithMask];
+      DNS = nameservers;
+      DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
+      IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC)
+      LinkLocalAddressing = "ipv6";
+    };
+    routes = [
+      {
+        Destination = "0.0.0.0/0";
+        Gateway = defaultGateway;
+      }
+      {
+        Destination = "::/0";
+        Gateway = defaultGateway6;
+        GatewayOnLink = true; # it's a gateway on local link.
+      }
+    ];
+    linkConfig.RequiredForOnline = "routable";
   };
 
   # This value determines the NixOS release from which the default

hosts/k8s/k3s-prod-1-master-1/default.nix

@@ -20,6 +20,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "prod-cluster-1.writefor.fun";
+
+    kubeletExtraArgs = [
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64"
+      "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112"
+    ];
   };
 in {
   imports =

hosts/k8s/k3s-prod-1-master-2/default.nix

@@ -18,6 +18,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "prod-cluster-1.writefor.fun";
+
+    kubeletExtraArgs = [
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64"
+      "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112"
+    ];
   };
 in {
   imports =

hosts/k8s/k3s-prod-1-master-3/default.nix

@@ -18,6 +18,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "prod-cluster-1.writefor.fun";
+
+    kubeletExtraArgs = [
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64"
+      "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112"
+    ];
   };
 in {
   imports =

hosts/k8s/k3s-prod-1-worker-1/default.nix

@@ -17,6 +17,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "prod-cluster-1.writefor.fun";
+
+    kubeletExtraArgs = [
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64"
+      "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112"
+    ];
   };
 in {
   imports =

hosts/k8s/k3s-prod-1-worker-2/default.nix

@@ -17,6 +17,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "prod-cluster-1.writefor.fun";
+
+    kubeletExtraArgs = [
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64"
+      "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112"
+    ];
   };
 in {
   imports =

hosts/k8s/k3s-prod-1-worker-3/default.nix

@@ -17,6 +17,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "prod-cluster-1.writefor.fun";
+
+    kubeletExtraArgs = [
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64"
+      "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112"
+    ];
   };
 in {
   imports =

hosts/k8s/k3s-test-1-master-1/default.nix

@@ -20,6 +20,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+
+    # kubeletExtraArgs = [
+    #   # IPv4 Private CIDR(full) - 172.16.0.0/12
+    #   # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+    #   # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+    #   "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64"
+    #   "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112"
+    # ];
   };
 in {
   imports =

hosts/k8s/k3s-test-1-master-2/default.nix

@@ -18,6 +18,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+
+    # kubeletExtraArgs = [
+    #   # IPv4 Private CIDR(full) - 172.16.0.0/12
+    #   # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+    #   # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+    #   "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64"
+    #   "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112"
+    # ];
   };
 in {
   imports =

hosts/k8s/k3s-test-1-master-3/default.nix

@@ -18,6 +18,14 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+
+    # kubeletExtraArgs = [
+    #   # IPv4 Private CIDR(full) - 172.16.0.0/12
+    #   # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+    #   # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+    #   "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64"
+    #   "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112"
+    # ];
   };
 in {
   imports =

hosts/k8s/kubevirt-shoryu/default.nix

@@ -29,6 +29,12 @@
       # when cpu-manager's static policy is enabled
       # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods
       "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi"
+
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64"
+      "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112"
     ];
     nodeLabels = [
       "node-purpose=kubevirt"

hosts/k8s/kubevirt-shushou/default.nix

@@ -26,6 +26,12 @@
       # when cpu-manager's static policy is enabled
       # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods
       "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi"
+
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64"
+      "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112"
     ];
     nodeLabels = [
       "node-purpose=kubevirt"

hosts/k8s/kubevirt-youko/default.nix

@@ -26,6 +26,12 @@
       # when cpu-manager's static policy is enabled
       # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods
       "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi"
+
+      # IPv4 Private CIDR(full) - 172.16.0.0/12
+      # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
+      # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
+      "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64"
+      "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112"
     ];
     nodeLabels = [
       "node-purpose=kubevirt"

lib/genK3sAgentModule.nix

@@ -8,6 +8,15 @@
   package = pkgs.k3s;
 in {
   environment.systemPackages = [package];
+
+  # Kernel modules required by cilium
+  boot.kernelModules = ["ip6_tables" "ip6table_mangle" "ip6table_raw" "ip6table_filter"];
+  networking.enableIPv6 = true;
+  networking.nat = {
+    enable = true;
+    enableIPv6 = true;
+  };
+
   services.k3s = {
     enable = true;
     inherit package tokenFile;

lib/genK3sServerModule.nix

@@ -34,6 +34,13 @@ in {
     dive # explore docker layers
   ];
 
+  # Kernel modules required by cilium
+  boot.kernelModules = ["ip6_tables" "ip6table_mangle" "ip6table_raw" "ip6table_filter"];
+  networking.enableIPv6 = true;
+  networking.nat = {
+    enable = true;
+    enableIPv6 = true;
+  };
   services.k3s = {
     enable = true;
     inherit package tokenFile clusterInit;

lib/genKubeVirtGuestModule.nix

@@ -4,7 +4,7 @@
   networking,
   ...
 }: let
-  inherit (networking) defaultGateway nameservers;
+  inherit (networking) defaultGateway defaultGateway6 nameservers;
   inherit (networking.hostsAddr.${hostName}) iface ipv4;
   ipv4WithMask = "${ipv4}/24";
 in {
@@ -18,19 +18,36 @@ in {
     "exfat"
   ];
 
-  networking = {inherit hostName;};
+  networking = {
+    inherit hostName;
+
+    # we use networkd instead
+    networkmanager.enable = false;
+    useDHCP = false;
+  };
   networking.useNetworkd = true;
   systemd.network.enable = true;
 
-  # Add ipv4 address to the bridge.
   systemd.network.networks."10-${iface}" = {
     matchConfig.Name = [iface];
     networkConfig = {
       Address = [ipv4WithMask];
-      Gateway = defaultGateway;
       DNS = nameservers;
-      IPv6AcceptRA = true;
+      DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
+      IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC)
+      LinkLocalAddressing = "ipv6";
     };
+    routes = [
+      {
+        Destination = "0.0.0.0/0";
+        Gateway = defaultGateway;
+      }
+      {
+        Destination = "::/0";
+        Gateway = defaultGateway6;
+        GatewayOnLink = true; # it's a gateway on local link.
+      }
+    ];
     linkConfig.RequiredForOnline = "routable";
   };
 

lib/genKubeVirtHostModule.nix

@@ -4,7 +4,9 @@
   networking,
   ...
 }: let
-  inherit (networking.hostsAddr.${hostName}) iface;
+  inherit (networking) defaultGateway defaultGateway6 nameservers;
+  inherit (networking.hostsAddr.${hostName}) iface ipv4;
+  ipv4WithMask = "${ipv4}/24";
 in {
   # supported file systems, so we can mount any removable disks with these filesystems
   boot.supportedFilesystems = [
@@ -31,15 +33,17 @@ in {
     # --- network --- #
     "net.bridge.bridge-nf-call-iptables" = 1;
     "net.core.somaxconn" = 32768;
-    "net.ipv4.ip_forward" = 1;
+
+    # ----- IPv4 ----- #
+    "net.ipv4.ip_forward" = 1; # Enable forwarding
     "net.ipv4.conf.all.forwarding" = 1;
     "net.ipv4.neigh.default.gc_thresh1" = 4096;
     "net.ipv4.neigh.default.gc_thresh2" = 6144;
     "net.ipv4.neigh.default.gc_thresh3" = 8192;
     "net.ipv4.neigh.default.gc_interval" = 60;
     "net.ipv4.neigh.default.gc_stale_time" = 120;
-
+    # ----- IPv6 ----- #
-    "net.ipv6.conf.all.disable_ipv6" = 1; # disable ipv6
+    "net.ipv6.conf.all.forwarding" = 1; # Enable forwarding
 
     # --- memory --- #
     "vm.swappiness" = 0; # don't swap unless absolutely necessary
@@ -67,6 +71,16 @@ in {
     enable = true;
   };
 
+  networking = {
+    inherit hostName;
+
+    # we use networkd instead
+    networkmanager.enable = false;
+    useDHCP = false;
+  };
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
   # Enable the Open vSwitch as a systemd service
   # It's required by kubernetes' ovs-cni plugin.
   virtualisation.vswitch = {
@@ -82,15 +96,40 @@ in {
       interfaces.${iface} = {};
     };
   };
-  networking = {
-    inherit hostName;
-    inherit (networking) defaultGateway nameservers;
 
-    networkmanager.enable = false;
+  # systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
-    # Set the host's address on the OVS bridge interface instead of the physical interface!
+
-    interfaces.ovsbr1 = networking.hostsInterface.${hostName}.interfaces.${iface};
+  # Set the host's address on the OVS bridge interface instead of the physical interface!
-    dhcpcd.enable = false; # disable dhcpcd, it's useless for the host
+  systemd.network.networks = {
-    enableIPv6 = true;
+    "10-ovsbr1" = {
+      matchConfig.Name = ["ovsbr1"];
+      networkConfig = {
+        Address = [ipv4WithMask];
+        DNS = nameservers;
+        DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
+        IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC)
+        LinkLocalAddressing = "ipv6";
+      };
+      routes = [
+        {
+          Destination = "0.0.0.0/0";
+          Gateway = defaultGateway;
+        }
+        {
+          Destination = "::/0";
+          Gateway = defaultGateway6;
+          GatewayOnLink = true; # it's a gateway on local link.
+        }
+      ];
+      linkConfig.RequiredForOnline = "routable";
+    };
+    "20-${iface}" = {
+      matchConfig.Name = [iface];
+      networkConfig.LinkLocalAddressing = "no";
+      # tell networkd ignore this interface.
+      # it's managed by openvswitch
+      linkConfig.RequiredForOnline = "no";
+    };
   };
 
   # This value determines the NixOS release from which the default