feat: monitoring - grafana dashboards, alertmanager alerting rules

hosts/idols_ruby/grafana/dashboards.yml

@@ -0,0 +1,20 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'Dashboards'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 20
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: false
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/dashboards
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true

hosts/idols_ruby/grafana/dashboards/README.md

@@ -0,0 +1,5 @@
+# Grafana Dashbaords
+
+1. https://grafana.com/grafana/dashboards/1860-node-exporter-full/
+2. https://grafana.com/grafana/dashboards/9578-alertmanager/
+

hosts/idols_ruby/grafana/datasources.yml

@@ -0,0 +1,21 @@
+# https://grafana.com/docs/grafana/latest/datasources/prometheus/
+apiVersion: 1
+
+datasources:
+  - name: prometheus-homelab
+    type: prometheus
+    access: proxy
+    # Access mode - proxy (server in the UI) or direct (browser in the UI).
+    url: http://localhost:9090
+    jsonData:
+      httpMethod: POST
+      manageAlerts: true
+      prometheusType: Prometheus
+      prometheusVersion: 2.49.0
+      cacheLevel: 'High'
+      disableRecordingRules: false
+      # As of Grafana 10, the Prometheus data source can be configured to query live dashboards 
+      # incrementally, instead of re-querying the entire duration on each dashboard refresh.
+      # Increasing the duration of the incrementalQueryOverlapWindow will increase the size of every incremental query, 
+      # but might be helpful for instances that have inconsistent results for recent data.
+      incrementalQueryOverlapWindow: 10m

hosts/idols_ruby/grafana/default.nix

@@ -49,4 +49,6 @@
       dashboards.path = ./dashboards.yml;
     };
   };
+
+  environment.etc."grafana/dashboards".source = ./dashboards;
 }