fix: victoriametrics (#172)

* chore: update aqua's host key

* fix: victoriametrics

* feat: adjust order of nix cache mirrors

* feat: update mysecrets

* fix: use bind mount - Failed at step STATE_DIRECTORY

hosts/idols-aquamarine/monitoring/module/default.nix

@@ -8,28 +8,23 @@ with lib; let
   cfg = config.services.my-victoriametrics;
   settingsFormat = pkgs.formats.yaml {};
 
-  workingDir = "/var/lib/" + cfg.stateDir;
-  startCommandLine =
-    lib.escapeShellArgs [
+  startCLIList =
+    [
       "${cfg.package}/bin/victoria-metrics"
-      "-storageDataPath=${workingDir}"
+      "-storageDataPath=/var/lib/${cfg.stateDir}"
       "-httpListenAddr=${cfg.listenAddress}"
       "-retentionPeriod=${cfg.retentionPeriod}"
     ]
-    ++ lib.optional (cfg.prometheusConfig != null) "-promscrape.config=${prometheusConfigYml}"
     ++ cfg.extraOptions;
   prometheusConfigYml = checkedConfig (
     settingsFormat.generate "prometheusConfig.yaml" cfg.prometheusConfig
   );
 
   checkedConfig = file:
-    if cfg.checkConfig
-    then
-      pkgs.runCommand "checked-config" {nativeBuildInputs = [cfg.package];} ''
-        ln -s ${file} $out
-        ${startCommandLine} -dryRun
-      ''
-    else file;
+    pkgs.runCommand "checked-config" {nativeBuildInputs = [cfg.package];} ''
+      ln -s ${file} $out
+      ${lib.escapeShellArgs startCLIList} -promscrape.config=${file} -dryRun
+    '';
 in {
   options.services.my-victoriametrics = {
     enable = mkEnableOption "VictoriaMetrics, a time series database.";
@@ -130,15 +125,17 @@ in {
       startLimitBurst = 5;
 
       serviceConfig = {
-        ExecStart = startCommandLine;
+        ExecStart = lib.escapeShellArgs (
+          startCLIList
+          ++ lib.optional (cfg.prometheusConfig != null) ["-promscrape.config=${prometheusConfigYml}"]
+        );
+
         DynamicUser = true;
         User = "victoriametrics";
-        Group = "victoriametrics";
         RestartSec = 1;
         Restart = "on-failure";
         RuntimeDirectory = "victoriametrics";
         RuntimeDirectoryMode = "0700";
-        WorkingDirectory = workingDir;
         StateDirectory = cfg.stateDir;
         StateDirectoryMode = "0700";
 

hosts/idols-aquamarine/monitoring/victoriametrics.nix

@@ -3,13 +3,23 @@
   myvars,
   ...
 }: {
+  # Since victoriametrics use DynamicUser, the user & group do not exists before the service starts.
+  # this group is used as a supplementary Unix group for the service to access our data dir(/data/apps/xxx)
+  users.groups.victoriametrics-data = {};
+
   # Workaround for victoriametrics to store data in another place
   # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
-    "D /data/apps/victoriametrics 0751 victoriametrics victoriametrics - -"
-    "L+ /var/lib/victoriametrics - - - - /data/apps/victoriametrics"
+    "D /data/apps/victoriametrics 0770 root victoriametrics-data - -"
   ];
 
+  # Symlinks do not work with DynamicUser, so we should use bind mount here.
+  # https://github.com/systemd/systemd/issues/25097#issuecomment-1929074961
+  systemd.services.victoriametrics.serviceConfig = {
+    SupplementaryGroups = ["victoriametrics-data"];
+    BindPaths = ["/data/apps/victoriametrics:/var/lib/victoriametrics:rbind"];
+  };
+
   # https://victoriametrics.io/docs/victoriametrics/latest/configuration/configuration/
   services.my-victoriametrics = {
     enable = true;