Merge pull request #104 from ryan4yin/migrate-services

feat: migrate some hosts to microvm

flake.nix

@@ -87,9 +87,9 @@
     # secrets management
     agenix = {
       # lock with git commit at 0.15.0
-      url = "github:ryantm/agenix/564595d0ad4be7277e07fa63b5a991b3c645655d";
+      # url = "github:ryantm/agenix/564595d0ad4be7277e07fa63b5a991b3c645655d";
       # replaced with a type-safe reimplementation to get a better error message and less bugs.
-      # url = "github:ryan4yin/ragenix";
+      url = "github:ryan4yin/ragenix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
@@ -108,8 +108,8 @@
 
     nuenv.url = "github:DeterminateSystems/nuenv";
 
-    # daeuniverse.url = "github:daeuniverse/flake.nix/unstable";
+    daeuniverse.url = "github:daeuniverse/flake.nix";
-    daeuniverse.url = "github:daeuniverse/flake.nix/exp";
+    # daeuniverse.url = "github:daeuniverse/flake.nix/exp";
 
     attic.url = "github:zhaofengli/attic";
 
@@ -118,6 +118,11 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    microvm = {
+      url = "github:astro/microvm.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     ########################  Some non-flake repositories  #########################################
 
     # AstroNvim is an aesthetic and feature-rich neovim config.

hosts/12kingdoms-suzu/default.nix

@@ -8,6 +8,8 @@
 #
 #  Suzu - Orange Pi 5 Plus, RK3588 + 16GB RAM
 #
+#  https://github.com/astro/microvm.nix
+#
 #############################################################
 let
   hostName = "suzu"; # Define your hostname.
@@ -19,14 +21,12 @@ in {
     ./hardware-configuration.nix
     ./disko-fs.nix
     ./impermanence.nix
+
+    ./networking.nix
+    ./microvm-host.nix
   ];
 
-  networking = {
+  networking = {inherit hostName;};
-    inherit hostName;
-    inherit (myvars.networking) defaultGateway nameservers;
-    inherit (myvars.networking.hostsInterface.${hostName}) interfaces;
-    networkmanager.enable = false;
-  };
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions

hosts/12kingdoms-suzu/microvm-host.nix

@@ -0,0 +1,92 @@
+{
+  myvars,
+  mylib,
+  daeuniverse,
+  agenix,
+  microvm,
+  mysecrets,
+  nuenv,
+  ...
+}: {
+  imports = [
+    # Include the microvm host module
+    microvm.nixosModules.host
+  ];
+
+  microvm.vms = {
+    suzi = {
+      autostart = true;
+      restartIfChanged = true;
+
+      specialArgs = {inherit myvars mylib daeuniverse agenix mysecrets nuenv;};
+
+      config = {
+        imports = [
+          ./router
+          ../../secrets/nixos.nix
+          ../../modules/nixos/base/ssh.nix
+          ../../modules/nixos/base/user-group.nix
+          ../../modules/base.nix
+        ];
+
+        modules.secrets.server.network.enable = true;
+
+        microvm = {
+          mem = 1024; # RAM allocation in MB
+          vcpu = 1; # Number of Virtual CPU cores
+
+          interfaces = [
+            {
+              type = "tap";
+              id = "vm-suzi"; # should be prefixed with "vm-"
+              mac = "02:00:00:00:00:01";
+            }
+          ];
+
+          # Block device images for persistent storage
+          # microvm use tmpfs for root(/), so everything else
+          # is ephemeral and will be lost on reboot.
+          #
+          # you can check this by running `df -Th` & `lsblk` in the VM.
+          volumes = [
+            {
+              mountPoint = "/var";
+              image = "var.img";
+              size = 512;
+            }
+            {
+              mountPoint = "/etc";
+              image = "etc.img";
+              size = 50;
+            }
+          ];
+
+          # shares can not be set to `neededForBoot = true;`
+          # so if you try to use a share in boot script(such as system.activationScripts), it will fail!
+          shares = [
+            {
+              # It is highly recommended to share the host's nix-store
+              # with the VMs to prevent building huge images.
+              # a host's /nix/store will be picked up so that no
+              # squashfs/erofs will be built for it.
+              #
+              # by this way, /nix/store is readonly in the VM,
+              # and thus the VM can't run any command that modifies
+              # the store. such as nix build, nix shell, etc...
+              # if you want to run nix commands in the VM, see
+              # https://github.com/astro/microvm.nix/blob/main/doc/src/shares.md#writable-nixstore-overlay
+              tag = "ro-store"; # Unique virtiofs daemon tag
+              proto = "virtiofs"; # virtiofs is faster than 9p
+              source = "/nix/store";
+              mountPoint = "/nix/.ro-store";
+            }
+          ];
+
+          hypervisor = "qemu";
+          # Control socket for the Hypervisor so that a MicroVM can be shutdown cleanly
+          socket = "control.socket";
+        };
+      };
+    };
+  };
+}

hosts/12kingdoms-suzu/microvm.md

@@ -0,0 +1,37 @@
+# microvm.nix
+
+## Commands
+
+> https://github.com/astro/microvm.nix/blob/main/doc/src/microvm-command.md
+
+```bash
+# list vm
+microvm -l
+
+# update vm
+microvm -u my-microvm
+
+
+# show logs of a vm
+journalctl -u microvm@my-microvm -n 50
+
+# stop vm
+systemctl stop microvm@$NAME
+
+# remove vm
+rm -rf /var/lib/microvms/$NAME
+
+# Run a MicroVM in foreground(for testing)
+# You have to stop the vm before running this command!
+microvm -r my-microvm
+
+# Stop a MicroVM that is running in foreground
+## 1. run `sudo shutdown -h now` in the vm
+## 2. run `systemctl stop microvm@my-microvm` in the host
+```
+
+## FAQ
+
+### 1. enter the vm without ssh
+
+[Enter running machine as systemd service](https://github.com/astro/microvm.nix/issues/123)

hosts/12kingdoms-suzu/networking.nix

@@ -0,0 +1,49 @@
+{myvars, ...}: let
+  hostName = "suzu";
+  inherit (myvars.networking) defaultGateway nameservers;
+  inherit (myvars.networking.hostsAddr.${hostName}) iface ipv4;
+
+  ipv4WithMask = "${ipv4}/24";
+in {
+  boot = {
+    kernel = {
+      sysctl = {
+        # forward network packets that are not destined for the interface on which they were received
+        "net.ipv4.conf.all.forwarding" = true;
+        "net.ipv6.conf.all.forwarding" = true;
+      };
+    };
+  };
+
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
+  # A bridge to link all VM's TAP interfaces into local network.
+  # https://github.com/astro/microvm.nix/blob/main/doc/src/simple-network.md
+  systemd.network.networks."10-lan" = {
+    # match on the main interface and all VM interfaces
+    matchConfig.Name = [iface "vm-*"];
+    networkConfig = {
+      Bridge = "br0";
+    };
+  };
+
+  systemd.network.netdevs."br0" = {
+    netdevConfig = {
+      Name = "br0";
+      Kind = "bridge";
+    };
+  };
+
+  # Add ipv4 address to the bridge.
+  systemd.network.networks."10-lan-bridge" = {
+    matchConfig.Name = "br0";
+    networkConfig = {
+      Address = [ipv4WithMask];
+      Gateway = defaultGateway;
+      DNS = nameservers;
+      IPv6AcceptRA = true;
+    };
+    linkConfig.RequiredForOnline = "routable";
+  };
+}

hosts/12kingdoms-suzu/router/README.md

@@ -0,0 +1,48 @@
+# Dae - NixOS Router
+
+A router(IPv4 only) with a transparent proxy to bypass the G|F|W.
+
+NOTE: dae do not provides a http/socks5 proxy server, so a v2ray server is running on
+[idols_kana](../idols_kana/proxy.nix) to provides a http/socks5 proxy service.
+
+## Troubleshooting
+
+### Can not access the global internet
+
+1. Check whether the subscription url is accessible.
+   - If not, then you need to get a new subscription url and update the `dae`'s configuration.
+1. Check the `dae` service's log by `journalctl -u dae -n 1000`.
+
+### DNS cannot be resolved
+
+1. `sudo systemctl stop dae`, then try to resolve the domain name again.
+   - If it works, the problem is caused by `dae` service.
+   - check dae's log by `journalctl -u dae -n 1000`
+1. DNS & DHCP is provided by `dnsmasq` service, check the configuration of `dnsmasq`.
+
+### DHCP cannot be obtained
+
+1. `ss -tunlp`, check if `dnsmasq` is running and listening on udp port 67.
+1. `journalctl -u dnsmasq -n 1000` to check the log of `dnsmasq`.
+1. Request a new IP address by disconnect and reconnect one of your devices' wifi.
+1. `nix shell nixpkgs#dhcpdump` and then `sudo dhcpdump -i br-lan`, check if the DHCP request is
+   received by `dnsmasq`.
+   1. The server listens on UDP port number 67, and the client listens on UDP port number 68.
+   1. DHCP operations fall into four phases:
+      1. Server **discovery**: The DHCP client broadcasts a DHCPDISCOVER message on the network
+         subnet using the destination address 255.255.255.255 (limited broadcast) or the specific
+         subnet broadcast address (directed broadcast).
+      1. IP lease **offer**: When a DHCP server receives a DHCPDISCOVER message from a client, which
+         is an IP address lease request, the DHCP server reserves an IP address for the client and
+         makes a lease offer by sending a DHCPOFFER message to the client.
+      1. IP lease **request**: In response to the DHCP offer, the client replies with a DHCPREQUEST
+         message, broadcast to the server,[a] requesting the offered address.
+      1. IP lease **acknowledgement**: When the DHCP server receives the DHCPREQUEST message from
+         the client, it sends a DHCPACK packet to the client, which includes the lease duration and
+         any other configuration information that the client might have requested.
+   1. So if you see only `DISCOVER` messages, the dhsmasq is not working properly.
+
+## References
+
+- <https://github.com/ghostbuster91/blogposts/blob/main/router2023-part2/main.md>
+- <https://github.com/ghostbuster91/nixos-router>

hosts/12kingdoms-suzu/router/config.dae

@@ -0,0 +1,320 @@
+# https://github.com/daeuniverse/dae/discussions/81
+# https://github.com/daeuniverse/dae/blob/main/example.dae
+
+# load all dae files placed in ./config.d/
+include {
+    config.d/*.dae
+}
+global {
+    ##### Software options.
+
+    # tproxy port to listen on. It is NOT a HTTP/SOCKS port, and is just used by eBPF program.
+    # In normal case, you do not need to use it.
+    tproxy_port: 12345
+
+    # Set it true to protect tproxy port from unsolicited traffic. Set it false to allow users to use self-managed
+    # iptables tproxy rules.
+    tproxy_port_protect: true
+
+    # If not zero, traffic sent from dae will be set SO_MARK. It is useful to avoid traffic loop with iptables tproxy
+    # rules.
+    so_mark_from_dae: 1
+
+    # Log level: error, warn, info, debug, trace.
+    log_level: info
+
+    # Disable waiting for network before pulling subscriptions.
+    disable_waiting_network: false
+
+
+    ##### Interface and kernel options.
+
+    # The LAN interface to bind. Use it if you want to proxy LAN.
+    # Multiple interfaces split by ",".
+    lan_interface: br-lan
+
+    # The WAN interface to bind. Use it if you want to proxy localhost.
+    # Multiple interfaces split by ",". Use "auto" to auto detect.
+    # 
+    # Disable this to avoid problems with the proxy server that prevent the subscription link from being updated
+    # wan_interface: auto
+
+    # Automatically configure Linux kernel parameters like ip_forward and send_redirects. Check out
+    # https://github.com/daeuniverse/dae/blob/main/docs/en/user-guide/kernel-parameters.md to see what will dae do.
+    auto_config_kernel_parameter: false
+
+    ##### Node connectivity check.
+
+    # Host of URL should have both IPv4 and IPv6 if you have double stack in local.
+    # First is URL, others are IP addresses if given.
+    # Considering traffic consumption, it is recommended to choose a site with anycast IP and less response.
+    #tcp_check_url: 'http://cp.cloudflare.com'
+    tcp_check_url: 'http://cp.cloudflare.com,1.1.1.1,2606:4700:4700::1111'
+
+    # The HTTP request method to `tcp_check_url`. Use 'HEAD' by default because some server implementations bypass
+    # accounting for this kind of traffic.
+    tcp_check_http_method: HEAD
+
+    # This DNS will be used to check UDP connectivity of nodes. And if dns_upstream below contains tcp, it also be used to check
+    # TCP DNS connectivity of nodes.
+    # First is URL, others are IP addresses if given.
+    # This DNS should have both IPv4 and IPv6 if you have double stack in local.
+    #udp_check_dns: 'dns.google.com:53'
+    udp_check_dns: 'dns.google.com:53,8.8.8.8,2001:4860:4860::8888'
+
+    check_interval: 30s
+
+    # Group will switch node only when new_latency <= old_latency - tolerance.
+    check_tolerance: 50ms
+
+
+    ##### Connecting options.
+
+    # Optional values of dial_mode are:
+    # 1. "ip". Dial proxy using the IP from DNS directly. This allows your ipv4, ipv6 to choose the optimal path
+    #       respectively, and makes the IP version requested by the application meet expectations. For example, if you
+    #       use curl -4 ip.sb, you will request IPv4 via proxy and get a IPv4 echo. And curl -6 ip.sb will request IPv6.
+    #       This may solve some weird full-cone problem if your are be your node support that. Sniffing will be disabled
+    #       in this mode.
+    # 2. "domain". Dial proxy using the domain from sniffing. This will relieve DNS pollution problem to a great extent
+    #       if have impure DNS environment. Generally, this mode brings faster proxy response time because proxy will
+    #       re-resolve the domain in remote, thus get better IP result to connect. This policy does not impact routing.
+    #       That is to say, domain rewrite will be after traffic split of routing and dae will not re-route it.
+    # 3. "domain+". Based on domain mode but do not check the reality of sniffed domain. It is useful for users whose
+    #       DNS requests do not go through dae but want faster proxy response time. Notice that, if DNS requests do not
+    #       go through dae, dae cannot split traffic by domain.
+    # 4. "domain++". Based on domain+ mode but force to re-route traffic using sniffed domain to partially recover
+    #       domain based traffic split ability. It doesn't work for direct traffic and consumes more CPU resources.
+    dial_mode: domain
+
+    # Allow insecure TLS certificates. It is not recommended to turn it on unless you have to.
+    allow_insecure: false
+
+    # Timeout to waiting for first data sending for sniffing. It is always 0 if dial_mode is ip. Set it higher is useful
+    # in high latency LAN network.
+    sniffing_timeout: 100ms
+
+    # TLS implementation. tls is to use Go's crypto/tls. utls is to use uTLS, which can imitate browser's Client Hello.
+    tls_implementation: tls
+
+    # The Client Hello ID for uTLS to imitate. This takes effect only if tls_implementation is utls.
+    # See more: https://github.com/daeuniverse/dae/blob/331fa23c16/component/outbound/transport/tls/utls.go#L17
+    utls_imitate: chrome_auto
+}
+
+# See https://github.com/daeuniverse/dae/blob/main/docs/en/configuration/dns.md for full examples.
+dns {
+    # For example, if ipversion_prefer is 4 and the domain name has both type A and type AAAA records, the dae will only
+    # respond to type A queries and response empty answer to type AAAA queries.
+    ipversion_prefer: 4
+
+    # Give a fixed ttl for domains. Zero means that dae will request to upstream every time and not cache DNS results
+    # for these domains.
+    #fixed_domain_ttl {
+    #    ddns.example.org: 10
+    #    test.example.org: 3600
+    #}
+
+    upstream {
+        # Value can be scheme://host:port, where the scheme can be tcp/udp/tcp+udp.
+        # If host is a domain and has both IPv4 and IPv6 record, dae will automatically choose
+        # IPv4 or IPv6 to use according to group policy (such as min latency policy).
+        # Please make sure DNS traffic will go through and be forwarded by dae, which is REQUIRED for domain routing.
+        # If dial_mode is "ip", the upstream DNS answer SHOULD NOT be polluted, so domestic public DNS is not recommended.
+
+        alidns: 'udp://223.5.5.5:53'
+        googledns: 'tcp+udp://8.8.8.8:53'
+    }
+    routing {
+        # According to the request of dns query, decide to use which DNS upstream.
+        # Match rules from top to bottom.
+        request {
+            # Lookup China mainland domains using alidns, otherwise googledns.
+            qname(geosite:cn) -> alidns
+            # fallback is also called default.
+            fallback: googledns
+
+            # other custom rules
+            qname(full:analytics.google.com) -> googledns # do not block google analytics(console)
+            qname(regex: '.+\.nixos.org$') -> googledns
+            qname(geosite:category-ads) -> reject
+            qname(geosite:category-ads-all) -> reject
+            qtype(aaaa) -> reject
+            qname(regex: '.+\.linkedin$') -> googledns
+        }
+
+        # According to the response of dns query, decide to accept or re-lookup using another DNS upstream.
+        # Match rules from top to bottom.
+        response {
+            # Trusted upstream. Always accept its result.
+            upstream(googledns) -> accept
+
+            # Possibly polluted(domain resolved to a private ip), re-lookup using googledns.
+            ip(geoip:private) && !qname(geosite:cn) -> googledns
+
+            fallback: accept
+        }
+    }
+}
+
+# Node group (outbound).
+group {
+    proxy {
+        filter: name(keyword: 'Hong Kong')
+        filter: name(keyword: '香港')
+        filter: name(keyword: 'Singapore')
+        filter: name(keyword: '新加坡')
+        # Filter nodes and give a fixed latency offset to archive latency-based failover.
+        # In this example, there is bigger possibility to choose US node even if original latency of US node is higher.
+        filter: name(keyword: 'USA') [add_latency: -500ms]
+        filter: name(keyword: '美国') [add_latency: -500ms]
+        filter: name(keyword: 'UK') [add_latency: -300ms]
+        # filter: name(keyword: '英国') [add_latency: -300ms]
+        # filter: name(keyword: 'Japan') [add_latency: 300ms]
+        # filter: name(keyword: '日本') [add_latency: 300ms]
+
+        # Other filters:
+        #   Filter nodes from the global node pool defined by the subscription and node section above.
+        #     filter: subtag(regex: '^my_', another_sub) && !name(keyword: 'ExpireAt:')
+        #   Filter nodes from the global node pool defined by tag.
+        #     filter: name('node_a','node_b')
+
+        # Select the node with min average of the last 10 latencies from the group for every connection.
+        policy: min_avg10
+        # Other policies:
+        #   random         - Randomly select a node from the group for every connection.
+        #   fixed(0)       - Select the first node from the group for every connection.
+        #   min            - Select the node with min last latency from the group for every connection.
+        #   min_moving_avg - Select the node with min moving average of latencies from the group for every connection.
+    }
+
+    media {
+        filter: name(keyword: 'Hong Kong')
+        filter: name(keyword: '香港')
+        filter: name(keyword: 'Singapore')
+        filter: name(keyword: '新加坡')
+        filter: name(keyword: 'USA') [add_latency: -500ms]
+        filter: name(keyword: '美国') [add_latency: -500ms]
+        filter: name(keyword: 'UK') [add_latency: -300ms]
+        filter: name(keyword: '英国') [add_latency: -300ms]
+        filter: name(keyword: 'Japan') [add_latency: 300ms]
+        filter: name(keyword: '日本') [add_latency: 300ms]
+
+        policy: min_avg10
+    }
+
+    ssh-proxy {
+        filter: name(keyword: 'UK')
+        filter: name(keyword: '英国')
+        policy: min_avg10
+    }
+
+    sg {
+        filter: name(keyword: 'Singapore')
+        filter: name(keyword: '新加坡')
+        policy: min_avg10
+    }
+
+    usa {
+        filter: name(keyword: 'USA')
+        filter: name(keyword: '美国')
+        policy: min_avg10
+    }
+}
+
+# See https://github.com/daeuniverse/dae/blob/main/docs/en/configuration/routing.md for full examples.
+# Pname has the highest priority, so should be placed in the front.
+# Priority of other rules is the same as the order of the rules defined in this file.
+routing {
+    ### Preset rules.
+
+    # Network managers in localhost should be direct to 
+    # avoid false negative network connectivity check when binding to WAN.
+    pname(NetworkManager) -> direct
+    pname(systemd-networkd) -> direct
+
+    # Put it in the front to prevent broadcast, multicast and other packets that should be sent to the LAN from being
+    # forwarded by the proxy.
+    # "dip" means destination IP.
+    dip(224.0.0.0/3, 'ff00::/8') -> direct
+
+    # This line allows you to access private addresses directly instead of via your proxy. If you really want to access
+    # private addresses in your proxy host network, modify the below line.
+    dip(geoip:private) -> direct
+
+    # --- Core rules ---#
+
+    # Disable HTTP3(QUIC) because it usually consumes too much cpu/mem resources.
+    l4proto(udp) && dport(443) -> block
+
+    # Direct access to all Chinese mainland-related IP addresses
+    dip(geoip:cn) -> direct
+    domain(geosite:cn) -> direct
+
+    # Block ads
+    domain(full:analytics.google.com) -> proxy  # do not block google analytics(console)
+    domain(geosite:category-ads) -> block
+    domain(geosite:category-ads-all) -> block
+
+    # DNS
+    dip(8.8.8.8, 8.8.4.4) -> proxy
+    dip(223.5.5.5, 223.6.6.6) -> direct
+    domain(full:dns.alidns.com) -> direct
+    domain(full:dns.googledns.com) -> proxy
+    domain(full:dns.opendns.com) -> proxy
+
+    # --- Rules for other commonly used sites ---#
+
+    # SSH - tcp port 22 is blocked by many proxy servers.
+    dport(22) && !dip(geoip:cn) && !domain(geosite:cn) -> ssh-proxy
+
+    ### OpenAI
+    domain(geosite:openai) -> sg
+    domain(regex:'.+\.openai$') -> sg
+
+    ### Media
+    domain(geosite:netflix) -> media
+
+    ### Proxy
+    domain(suffix: linkedin.com) -> proxy
+    domain(keyword:'linkedin') -> proxy
+    domain(regex:'.+\.linkedin\.com$') -> proxy
+    domain(regex:'.+\.quay\.io$') -> proxy
+    domain(regex:'.+\.notion\.so$') -> proxy
+    domain(regex:'.+\.amazon\.com$') -> proxy
+    domain(regex:'.+\.oracle\.com$') -> proxy
+    domain(regex:'.+\.docker\.com$') -> proxy
+    domain(regex:'.+\.kubernetes\.io$') -> proxy
+    domain(regex:'.+\.nixos\.org$') -> proxy
+
+    domain(geosite:microsoft) -> proxy
+    domain(geosite:linkedin) -> proxy
+    domain(geosite:twitter) -> proxy
+    domain(geosite:telegram) -> proxy
+    domain(geosite:google) -> proxy
+    domain(geosite:apple) -> proxy
+    domain(geosite:category-container) -> proxy
+    domain(geosite:category-dev) -> proxy
+    domain(geosite:google-scholar) -> proxy
+    domain(geosite:category-scholar-!cn) -> proxy
+
+    ### Direct
+    domain(regex:'.+\.edu\.cn$') -> direct
+    domain(keyword:'baidu') -> direct
+    domain(keyword:'bilibili') -> direct
+    domain(keyword:'taobao') -> direct
+    domain(keyword:'alibabadns') -> direct
+    domain(keyword:'alicdn') -> direct
+    domain(keyword:'tbcache') -> direct
+    domain(keyword:'zhihu') -> direct
+    domain(keyword:'douyu') -> direct
+    domain(geosite:cloudflare-cn) -> direct
+
+    # --- Fallback rules ---#
+
+    # Access all other foreign sites
+    domain(geosite:geolocation-!cn) -> proxy
+    !dip(geoip:cn) -> proxy
+
+    fallback: direct
+}

hosts/12kingdoms-suzu/router/dae.nix

@@ -0,0 +1,56 @@
+{
+  config,
+  pkgs,
+  daeuniverse,
+  ...
+}:
+# https://github.com/daeuniverse/flake.nix
+let
+  daeConfigPath = "/etc/dae/config.dae";
+  subscriptionConfigPath = "/etc/dae/config.d/subscription.dae";
+in {
+  imports = [
+    daeuniverse.nixosModules.dae
+  ];
+
+  # dae - eBPF-based Linux high-performance transparent proxy.
+  services.dae = {
+    enable = true;
+    package = daeuniverse.packages.${pkgs.system}.dae;
+    disableTxChecksumIpGeneric = false;
+    configFile = daeConfigPath;
+    assets = with pkgs; [v2ray-geoip v2ray-domain-list-community];
+    # alternatively, specify assets dir
+    # assetsPath = "/etc/dae";
+    openFirewall = {
+      enable = true;
+      port = 12345;
+    };
+  };
+
+  # dae supports two types of subscriptions: base64 encoded proxies, and sip008.
+  # subscription can be a url return the subscription, or a file path that contains the subscription.
+  #
+  # Nix decrypt and merge my dae's base config and subscription config here.
+  # the subscription config is something like:
+  #   ```
+  #   subscription {
+  #     'https://www.example.com/subscription/link'
+  #     'https://example.com/no_tag_link'
+  #   }
+  #   node {
+  #     # Support socks5, http, https, ss, ssr, vmess, vless, trojan, trojan-go, tuic, juicity
+  #     node_a: 'trojan://'
+  #     node_b: 'trojan://'
+  #     node_c: 'vless://'
+  #     node_d: 'vless://'
+  #     node_e: 'vmess://'
+  #     node_f: 'tuic://'
+  #     node_h: 'juicity://'
+  #   }
+  #   ```
+  system.activationScripts.installDaeConfig = ''
+    install -Dm 600 ${./config.dae} ${daeConfigPath}
+    install -Dm 600 ${config.age.secrets."dae-subscription.dae".path} ${subscriptionConfigPath}
+  '';
+}

hosts/12kingdoms-suzu/router/default.nix

@@ -0,0 +1,3 @@
+{mylib, ...}: {
+  imports = mylib.scanPaths ./.;
+}

hosts/12kingdoms-suzu/router/networking.nix

@@ -0,0 +1,184 @@
+{
+  lib,
+  myvars,
+  ...
+}: let
+  hostName = "suzi";
+  inherit (myvars.networking) mainGateway nameservers;
+  inherit (myvars.networking.hostsAddr.${hostName}) ipv4;
+
+  ipv4WithMask = "${ipv4}/24";
+  dhcpRange = {
+    start = "192.168.5.5";
+    end = "192.168.5.99";
+  };
+in {
+  # https://github.com/ghostbuster91/blogposts/blob/main/router2023-part2/main.md
+  boot = {
+    kernel = {
+      # https://github.com/daeuniverse/dae/blob/main/docs/en/user-guide/kernel-parameters.md
+      sysctl = {
+        # forward network packets that are not destined for the interface on which they were received
+        "net.ipv4.conf.all.forwarding" = true;
+        "net.ipv6.conf.all.forwarding" = true;
+        "net.ipv4.conf.br-lan.rp_filter" = 1;
+        "net.ipv4.conf.br-lan.send_redirects" = 0;
+      };
+    };
+  };
+
+  # Docker uses iptables internally to setup NAT for containers.
+  # This module disables the ip_tables kernel module, which is required for nftables to work.
+  # So make sure to disable docker here.
+  virtualisation.docker.enable = lib.mkForce false;
+  networking = {
+    useNetworkd = true;
+
+    useDHCP = false;
+    networkmanager.enable = false;
+    wireless.enable = false; # Enables wireless support via wpa_supplicant.
+    # No local firewall.
+    nat.enable = false;
+    firewall.enable = false;
+
+    # https://github.com/NixOS/nixpkgs/blob/nixos-23.11/nixos/modules/services/networking/nftables.nix
+    nftables = {
+      enable = true;
+      # Check the applied rules with `nft -a list ruleset`.
+      # Since this is a internal bypass router, we don't need to do NAT & can forward all traffic.
+      ruleset = ''
+        # Check out https://wiki.nftables.org/ for better documentation.
+        # Table for both IPv4 and IPv6.
+        table inet filter {
+          chain input {
+            type filter hook input priority 0;
+
+            # accept any localhost traffic
+            iifname lo accept
+
+            # accept any lan traffic
+            iifname br-lan accept
+
+            # count and drop any other traffic
+            counter drop
+          }
+
+          # Allow all outgoing connections.
+          chain output {
+            type filter hook output priority 0;
+            accept
+          }
+
+          # Allow all forwarding all traffic.
+          chain forward {
+            type filter hook forward priority 0;
+            accept
+          }
+        }
+      '';
+    };
+  };
+
+  # https://nixos.wiki/wiki/Systemd-networkd
+  systemd.network = {
+    netdevs = {
+      # Create the bridge interface
+      "20-br-lan" = {
+        netdevConfig = {
+          Kind = "bridge";
+          Name = "br-lan";
+        };
+      };
+    };
+    # This is a bypass router, so we do not need a wan interface here.
+    networks = {
+      "30-lan0" = {
+        # match the interface by type
+        matchConfig.Type = "ether";
+        # Connect to the bridge
+        networkConfig = {
+          Bridge = "br-lan";
+          ConfigureWithoutCarrier = true;
+        };
+        linkConfig.RequiredForOnline = "enslaved";
+      };
+      # Configure the bridge device we just created
+      "40-br-lan" = {
+        matchConfig.Name = "br-lan";
+        address = [
+          # configure addresses including subnet mask
+          ipv4WithMask # forwards all traffic to the gateway except for the router address itself
+        ];
+        routes = [
+          # forward all traffic to the main gateway
+          {routeConfig.Gateway = mainGateway;}
+        ];
+        bridgeConfig = {};
+        linkConfig.RequiredForOnline = "routable";
+      };
+    };
+  };
+
+  # resolved is conflict with dnsmasq
+  services.resolved.enable = false;
+  services.dnsmasq = {
+    enable = true;
+    # resolve local queries (add 127.0.0.1 to /etc/resolv.conf)
+    resolveLocalQueries = true; # may be conflict with dae, disable this.
+    alwaysKeepRunning = true;
+    # https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=tree
+    settings = {
+      # upstream DNS servers
+      server = nameservers;
+      # forces dnsmasq to try each query with each server strictly
+      # in the order they appear in the config.
+      strict-order = true;
+
+      # Never forward plain names (without a dot or domain part)
+      domain-needed = true;
+      # Never forward addresses in the non-routed address spaces(e.g. private IP).
+      bogus-priv = true;
+      # don't needlessly read /etc/resolv.conf which only contains the localhost addresses of dnsmasq itself.
+      no-resolv = true;
+
+      # Cache dns queries.
+      cache-size = 1000;
+
+      dhcp-range = ["${dhcpRange.start},${dhcpRange.end},24h"];
+      interface = "br-lan";
+      dhcp-sequential-ip = true;
+      dhcp-option = [
+        # Override the default route supplied by dnsmasq, which assumes the
+        # router is the same machine as the one running dnsmasq.
+        "option:router,${ipv4}"
+        "option:dns-server,${ipv4}"
+      ];
+
+      # local domains
+      local = "/lan/";
+      domain = "lan";
+      expand-hosts = true;
+
+      # don't use /etc/hosts
+      no-hosts = true;
+      address = [
+        # "/surfer.lan/192.168.10.1"
+      ];
+    };
+  };
+
+  # monitoring with prometheus
+  # https://github.com/NixOS/nixpkgs/blob/nixos-23.11/nixos/modules/services/monitoring/prometheus/exporters/dnsmasq.nix
+  services.prometheus.exporters.dnsmasq = {
+    enable = true;
+    listenAddress = "0.0.0.0";
+    port = 9153;
+    openFirewall = false;
+    leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
+  };
+
+  # The service irqbalance is useful as it assigns certain IRQ calls to specific CPUs instead of
+  # letting the first CPU core to handle everything.
+  # This is supposed to increase performance by hitting CPU cache more often.
+  services.irqbalance.enable = false;
+}

outputs/aarch64-linux/src/12kingdoms-suzu.nix

@@ -16,11 +16,16 @@
   ssh-user = "root";
 
   modules = {
-    nixos-modules = map mylib.relativeToRoot [
+    nixos-modules =
-      "modules/nixos/server/server-aarch64.nix"
+      (map mylib.relativeToRoot [
-      # host specific modules
+        "secrets/nixos.nix"
-      "hosts/12kingdoms-${name}"
+        "modules/nixos/server/server-aarch64.nix"
-    ];
+        # host specific modules
+        "hosts/12kingdoms-${name}"
+      ])
+      ++ [
+        {modules.secrets.server.network.enable = true;}
+      ];
   };
 
   inherit (inputs) nixos-rk3588;

vars/networking.nix

@@ -74,6 +74,10 @@
       iface = "enP4p65s0";
       ipv4 = "192.168.5.179";
     };
+    suzi = {
+      iface = "eth1";
+      ipv4 = "192.168.5.178";
+    };
 
     # ============================================
     # Kubernetes Clusters