feat: add distributed-builds & many hosts

feat: remove nur & devenv feat: adjust the structure to make it suitable for servers feat: add iso/proxmox generators and add docs about it feat: update ryan's openssh keys & add hashedPassword feat: add proxmox's nodes into ssh_config, with alias
2026-04-01 07:03:08 +02:00 · 2023-06-17 03:07:59 +08:00
parent a5f0922ba7
commit 45a7973553
48 changed files with 736 additions and 685 deletions
--- a/modules/darwin/core.nix
+++ b/modules/darwin/core.nix
@@ -1,6 +1,6 @@
 # all the configuration options are documented here:
 #    https://daiderd.com/nix-darwin/manual/index.html#sec-options
-{ pkgs, lib, ... }:
+{ pkgs, ... }:
 {
  # # enable flakes globally
  nix.settings.experimental-features = [ "nix-command" "flakes" ];
--- a/modules/nixos/core-desktop.nix
+++ b/modules/nixos/core-desktop.nix
@@ -1,47 +1,30 @@
-{ config, pkgs, devenv, ... }:
+{ lib, pkgs, ... }:

 {
-  # for nix server, we do not need to keep too much generations
-  boot.loader.systemd-boot.configurationLimit = 10;
-  # boot.loader.grub.configurationLimit = 10;
-  # do garbage collection weekly to keep disk usage low
-  nix.gc = {
-    automatic = true;
-    dates = "weekly";
-    options = "--delete-older-than 1w";
-  };

-  # Manual optimise storage: nix-store --optimise
-  # https://nixos.org/manual/nix/stable/command-ref/conf-file.html#conf-auto-optimise-store
-  nix.settings.auto-optimise-store = true;
+  imports = [
+    ./core-server.nix
+  ];

-  # enable flakes globally
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-
-  # Allow unfree packages
-  nixpkgs.config.allowUnfree = true;
-
-  # Set your time zone.
-  time.timeZone = "Asia/Shanghai";
-
-  # Select internationalisation properties.
-  i18n.defaultLocale = "en_US.UTF-8";
-
-  i18n.extraLocaleSettings = {
-    LC_ADDRESS = "zh_CN.UTF-8";
-    LC_IDENTIFICATION = "zh_CN.UTF-8";
-    LC_MEASUREMENT = "zh_CN.UTF-8";
-    LC_MONETARY = "zh_CN.UTF-8";
-    LC_NAME = "zh_CN.UTF-8";
-    LC_NUMERIC = "zh_CN.UTF-8";
-    LC_PAPER = "zh_CN.UTF-8";
-    LC_TELEPHONE = "zh_CN.UTF-8";
-    LC_TIME = "zh_CN.UTF-8";
-  };
+  # to install chrome, you need to enable unfree packages
+  nixpkgs.config.allowUnfree = lib.mkForce true;

  # Enable CUPS to print documents.
  services.printing.enable = true;

+  # DO NOT promote ryan to input password for `nix-store` and `nix-copy-closure`
+  security.sudo.extraRules = [
+    { users = [ "ryan" ];
+      commands = [
+        { command = "/run/current-system/sw/bin/nix-store" ;
+          options = [ "NOPASSWD" ];
+        }
+        { command = "/run/current-system/sw/bin/nix-copy-closure" ;
+          options = [ "NOPASSWD" ];
+        }
+      ];
+    }
+  ];

  # all fonts are linked to /nix/var/nix/profiles/system/sw/share/X11/fonts
  fonts = {
@@ -95,6 +78,7 @@
    };
  };

+  # dconf is a low-level configuration system.
  programs.dconf.enable = true;

  # networking.firewall.allowedTCPPorts = [ ... ];
@@ -121,20 +105,12 @@
  # List packages installed in system profile. To search, run:
  # $ nix search wget
  environment.systemPackages = with pkgs; [
-    neovim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.
-    wget
-    curl
-    git # used by nix flakes
-    git-lfs # used by huggingface models
-
-    devenv.packages."${pkgs.system}".devenv
-
    # the vscode insiders is designed to run alongside the main build, 
    # with a separate code-insiders command and a different config path
    # 
    # TODO install vscode into systemPackages to avoid binary collision error temporarily
    #     has collision between vscode & vscode-insider - /lib/vscode/chrome_crashpad_handler
-    ((pkgs.vscode.override { isInsiders = true; }).overrideAttrs (oldAttrs: {
+    ((vscode.override { isInsiders = true; }).overrideAttrs (oldAttrs: {
      src = (builtins.fetchTarball {
        url = "https://code.visualstudio.com/sha/download?build=insider&os=linux-x64";
        # you need to update this sha256 every time you update vscode insiders
@@ -143,23 +119,8 @@
      });
      version = "latest";
    }))
-
-    # create a fhs environment by command `fhs`, so we can run non-nixos packages in nixos!
-    (
-      let base = pkgs.appimageTools.defaultFhsEnvArgs; in
-      pkgs.buildFHSUserEnv (base // {
-        name = "fhs";
-        targetPkgs = pkgs: (base.targetPkgs pkgs) ++ [ pkgs.pkg-config ];
-        profile = "export FHS=1";
-        runScript = "bash";
-        extraOutputsToInstall = [ "dev" ];
-      })
-    )
  ];

-  # replace default editor with neovim
-  environment.variables.EDITOR = "nvim";
-
  # PipeWire is a new low-level multimedia framework.
  # It aims to offer capture and playback for both audio and video with minimal latency.
  # It support for PulseAudio-, JACK-, ALSA- and GStreamer-based applications. 
@@ -227,7 +188,6 @@
  # see https://github.com/NixOS/nixpkgs/blob/nixos-unstable/nixos/modules/programs/adb.nix
  programs.adb.enable = true;

-
  xdg.portal = {
    enable = true;
    wlr.enable = true;
@@ -252,7 +212,4 @@
  ];
  # set user's default shell system-wide
  users.defaultUserShell = pkgs.nushell;
-
-  # for power management
-  services.upower.enable = true;
 }
--- a/modules/nixos/core-server.nix
+++ b/modules/nixos/core-server.nix
@@ -1,14 +1,15 @@
-{ config, pkgs, devenv, ... }:
+{ lib, pkgs, ... }:

 {
+
  # for nix server, we do not need to keep too much generations
-  boot.loader.systemd-boot.configurationLimit = 10;
+  boot.loader.systemd-boot.configurationLimit = lib.mkDefault 10;
  # boot.loader.grub.configurationLimit = 10;
  # do garbage collection weekly to keep disk usage low
  nix.gc = {
-    automatic = true;
-    dates = "weekly";
-    options = "--delete-older-than 1w";
+    automatic = lib.mkDefault true;
+    dates = lib.mkDefault "weekly";
+    options = lib.mkDefault "--delete-older-than 1w";
  };

  # Manual optimise storage: nix-store --optimise
@@ -17,9 +18,11 @@

  # enable flakes globally
  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  
+  nix.settings.trusted-users = ["ryan"];

  # Allow unfree packages
-  nixpkgs.config.allowUnfree = true;
+  nixpkgs.config.allowUnfree = lib.mkDefault false;

  # Set your time zone.
  time.timeZone = "Asia/Shanghai";
@@ -39,45 +42,10 @@
    LC_TIME = "zh_CN.UTF-8";
  };

-  # all fonts are linked to /nix/var/nix/profiles/system/sw/share/X11/fonts
-  fonts = {
-    # use fonts specified by user rather than default ones
-    enableDefaultFonts = false;
-    fontDir.enable = true;
-
-    fonts = with pkgs; [
-      # icon fonts
-      material-design-icons
-      font-awesome
-
-      # 思源系列字体是 Adobe 主导的。其中汉字部分被称为「思源黑体」和「思源宋体」，是由 Adobe + Google 共同开发的
-      source-sans # 无衬线字体，不含汉字。字族名叫 Source Sans 3 和 Source Sans Pro，以及带字重的变体，加上 Source Sans 3 VF
-      source-han-sans # 思源黑体
-
-      # nerdfonts
-      (nerdfonts.override {
-        fonts = [
-          "FiraCode"
-          "JetBrainsMono"
-          "Iosevka"
-        ];
-      })
-    ];
-
-    # user defined fonts
-    # the reason there's Noto Color Emoji everywhere is to override DejaVu's
-    # B&W emojis that would sometimes show instead of some Color emojis
-    fontconfig.defaultFonts = {
-      sansSerif = [ "Noto Sans" "Noto Color Emoji" ];
-      monospace = [ "JetBrainsMono Nerd Font" "Noto Color Emoji" ];
-      emoji = [ "Noto Color Emoji" ];
-    };
-  };
-
  # networking.firewall.allowedTCPPorts = [ ... ];
  # networking.firewall.allowedUDPPorts = [ ... ];
  # Or disable the firewall altogether.
-  networking.firewall.enable = false;
+  networking.firewall.enable = lib.mkDefault false;

  # Enable the OpenSSH daemon.
  services.openssh = {
@@ -99,11 +67,28 @@
    aria2
    git # used by nix flakes
    git-lfs # used by huggingface models
+    k9s
+
+    # create a fhs environment by command `fhs`, so we can run non-nixos packages in nixos!
+    (
+      let base = pkgs.appimageTools.defaultFhsEnvArgs; in
+      pkgs.buildFHSUserEnv (base // {
+        name = "fhs";
+        targetPkgs = pkgs: (base.targetPkgs pkgs) ++ [ pkgs.pkg-config ];
+        profile = "export FHS=1";
+        runScript = "bash";
+        extraOutputsToInstall = [ "dev" ];
+      })
+    )
  ];

  # replace default editor with neovim
  environment.variables.EDITOR = "nvim";

+  virtualisation.docker = {
+    enable = true;
+  };
+
  # for power management
  services.power-profiles-daemon = {
    enable = true;
--- a/modules/nixos/gui-apps.nix
+++ b/modules/nixos/gui-apps.nix
@@ -1,17 +0,0 @@
-{ config, pkgs, ... }:
-
-{
-  # this params has problem with home-manager,
-  # so defined as NixOS Module here.
-  # nixpkgs.config.permittedInsecurePackages = [
-  #   "electron-19.0.7"  # required by wechat-uos, and it's already EOL
-  #   "openssl-1.1.1u"   # OpenSSL 1.1 is reaching its end of life on 2023/09/11
-  # ];
-
-  environment.systemPackages = [
-    # packages from nur-xddxdd
-    # config.nur.repos.xddxdd.wechat-uos
-  ];
-
-  # flatpack is recommended to install other apps such as netease-cloud-music/qqmusic/...
-}
--- a/modules/nixos/proxmox-hardware-configuration.nix
+++ b/modules/nixos/proxmox-hardware-configuration.nix
@@ -0,0 +1,67 @@
+{ config, lib, ... }:
+
+##############################################################################
+#
+#  Template for Proxmox's VM, mainly based on:
+#    https://github.com/NixOS/nixpkgs/blob/nixos-unstable/nixos/modules/virtualisation/proxmox-image.nix
+#
+#  the url above is used by `nixos-generator` to generate the Proxmox's VMA image file.
+# 
+##############################################################################
+
+let
+  bios = "seabios";
+  partitionTableType = if bios == "seabios" then "legacy" else "efi";
+  supportEfi = partitionTableType == "efi" || partitionTableType == "hybrid";
+  supportBios = partitionTableType == "legacy" || partitionTableType == "hybrid" || partitionTableType == "legacy+gpt";
+  hasBootPartition = partitionTableType == "efi" || partitionTableType == "hybrid";
+  hasNoFsPartition = partitionTableType == "hybrid" || partitionTableType == "legacy+gpt";
+in
+{
+
+  # DO NOT promote ryan to input password for sudo.
+  # this is a workaround for the issue of remote deploy:
+  #   https://github.com/NixOS/nixpkgs/issues/118655
+  security.sudo.extraRules = [
+    { users = [ "ryan" ];
+      commands = [
+        { command = "ALL" ;
+          options = [ "NOPASSWD" ];
+        }
+      ];
+    }
+  ];
+
+  boot = {
+    # after resize the disk, it will grow partition automatically.
+    growPartition = true;
+    kernelParams = [ "console=ttyS0" ];
+    loader.grub = {
+      device = lib.mkDefault (if (hasNoFsPartition || supportBios) then
+        # Even if there is a separate no-fs partition ("/dev/disk/by-partlabel/no-fs" i.e. "/dev/vda2"),
+        # which will be used the bootloader, do not set it as loader.grub.device.
+        # GRUB installation fails, unless the whole disk is selected.
+        "/dev/vda"
+      else
+        "nodev");
+      efiSupport = lib.mkDefault supportEfi;
+      efiInstallAsRemovable = lib.mkDefault supportEfi;
+    };
+
+    loader.timeout = 0;
+    initrd.availableKernelModules = [ "uas" "virtio_blk" "virtio_pci" ];
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/nixos";
+    autoResize = true;
+    fsType = "ext4";
+  };
+  fileSystems."/boot" = lib.mkIf hasBootPartition {
+    device = "/dev/disk/by-label/ESP";
+    fsType = "vfat";
+  };
+
+  # it alse had qemu-guest-agent installed by default.
+  services.qemuGuest.enable = lib.mkDefault true;
+}
--- a/modules/nixos/remote-building.nix
+++ b/modules/nixos/remote-building.nix
@@ -0,0 +1,112 @@
+{ ... }: {
+
+  ############################################
+  #
+  #  NixOS's Configuration for Remote Building / Distributed Building
+  #
+  #  Related Docs:
+  #    1. https://github.com/NixOS/nix/issues/7380
+  #    2. https://nixos.wiki/wiki/Distributed_build
+  ############################################
+
+  # set local's max-job to 0 to force remote building(disable local building)
+  # nix.settings.max-jobs = 0;
+  nix.distributedBuilds = true;
+  nix.buildMachines = 
+    let
+      sshUser = "ryan";
+      # ssh key's path on local machine
+      sshKey = "/home/ryan/.ssh/ai-idols";
+      systems = [
+        # native arch
+        "x86_64-linux"
+
+        # emulated arch using binfmt_misc and qemu-user
+        "aarch64-linux"
+        "riscv64-linux"
+      ];
+      # all available system features are poorly documentd here:
+      #  https://github.com/NixOS/nix/blob/e503ead/src/libstore/globals.hh#L673-L687
+      supportedFeatures = [
+        "benchmark"
+        "big-parallel"
+        "kvm"
+      ];
+    in
+      [
+        {
+          # some of my remote builders are running NixOS
+          # and has the same sshUser, sshKey, systems, etc.
+          inherit sshUser sshKey systems supportedFeatures;
+
+          # the hostName should be:
+          #   1. a hostname that can be resolved by DNS
+          #   2. the ip address of the remote builder
+          #   3. a host alias defined globally in /etc/ssh/ssh_config
+          hostName = "aquamarine";
+          # remote builder's max-job
+          maxJobs = 5;
+          # speedFactor's a signed integer
+          # nix seems always try to build on the machine with the highest speedFactor
+          speedFactor = 0;
+        }
+        {
+          inherit sshUser sshKey systems supportedFeatures;
+          hostName = "ruby";
+          maxJobs = 2;
+          speedFactor = 0;
+        }
+        {
+          inherit sshUser sshKey systems supportedFeatures;
+          hostName = "kana";
+          maxJobs = 1;
+          speedFactor = 0;
+        }
+      ];
+  # optional, useful when the builder has a faster internet connection than yours
+	nix.extraOptions = ''
+		builders-use-substitutes = true
+	'';
+
+  # define the host alias for remote builders
+  # this config will be written to /etc/ssh/ssh_config
+  programs.ssh.extraConfig = ''
+    Host ai
+      HostName 192.168.5.100
+      Port 22
+    
+    Host aquamarine
+      HostName 192.168.5.101
+      Port 22
+    
+    Host ruby
+      HostName 192.168.5.102
+      Port 22
+
+    Host kana
+      HostName 192.168.5.103
+      Port 22
+  '';
+
+  # define the host key for remote builders so that nix can verify all the remote builders
+  # this config will be written to /etc/ssh/ssh_known_hosts
+  programs.ssh.knownHosts = {
+    # 星野 愛久愛海, Hoshino Aquamarine
+    aquamarine = {
+      hostNames = [ "aquamarine" ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDnCQXlllHoLX5EvU+t6yP/npsmuxKt0skHVeJashizE";
+    };
+
+    # 星野 瑠美衣, Hoshino Rubii
+    ruby = {
+      hostNames = [ "ruby" ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICoIvsorGgnYpyunNrJfZ6Nyue7wBTx6LsyMOMrpgHJ/";
+    };
+
+    # 有馬 かな, Arima Kana
+    kana = {
+      hostNames = [ "kana" ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEuoNB0OSjCFyS022e+ZQCIsb/nhkw/XcWjksqK3sh5x";
+    };
+  };
+}
--- a/modules/nixos/user-group.nix
+++ b/modules/nixos/user-group.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, ... }:
+{ ... }:

 {
  users.groups = {
@@ -8,12 +8,14 @@
  };
  # Define a user account. Don't forget to set a password with ‘passwd’.
  users.users.ryan = {
+    # the hashed password with salt is generated by run `mkpasswd`.
+    hashedPassword = "$y$j9T$YQu5vhlnogjDFDWp9QkPh0$Eu85OiwllqvLg5fzRVMLVHNO7InA3ro8grTJJIepyH1";
    home = "/home/ryan";
    isNormalUser = true;
    description = "ryan";
    extraGroups = [ "ryan" "users" "networkmanager" "wheel" "docker" "wireshark" "adbusers" ];
    openssh.authorizedKeys.keys = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJx3Sk20pLL1b2PPKZey2oTyioODrErq83xG78YpFBoj"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDiipi59EnVbi6bK1bGrcbfEM263wgdNfbrt6VBC1rHx ryan@ai-idols"
    ];
  };
 }