nix-config/hosts/idols-aquamarine/monitoring/recoding_rules/node-exporter.yml

groups:
  - name: kube-prometheus-node-recording.rules
    rules:
      - expr:
          sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY
          (instance)
        record: instance:node_cpu:rate:sum
      - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
        record: instance:node_network_receive_bytes:rate:sum
      - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
        record: instance:node_network_transmit_bytes:rate:sum
      - expr:
          sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT
          (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance,
          cpu)) BY (instance)
        record: instance:node_cpu:ratio
      - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
        record: cluster:node_cpu:sum_rate5m
      - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))
        record: cluster:node_cpu:ratio

  - name: node-exporter.rules
    rules:
      - expr: |-
          count without (cpu, mode) (
            node_cpu_seconds_total{job="node-exporter",mode="idle"}
          )
        record: instance:node_num_cpu:sum
      - expr: |-
          1 - avg without (cpu) (
            sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m]))
          )
        record: instance:node_cpu_utilisation:rate5m
      - expr: |-
          (
            node_load1{job="node-exporter"}
          /
            instance:node_num_cpu:sum{job="node-exporter"}
          )
        record: instance:node_load1_per_cpu:ratio
      - expr: |-
          1 - (
            (
              node_memory_MemAvailable_bytes{job="node-exporter"}
              or
              (
                node_memory_Buffers_bytes{job="node-exporter"}
                +
                node_memory_Cached_bytes{job="node-exporter"}
                +
                node_memory_MemFree_bytes{job="node-exporter"}
                +
                node_memory_Slab_bytes{job="node-exporter"}
              )
            )
          /
            node_memory_MemTotal_bytes{job="node-exporter"}
          )
        record: instance:node_memory_utilisation:ratio
      - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m])
        record: instance:node_vmstat_pgmajfault:rate5m
      - expr:
          rate(node_disk_io_time_seconds_total{job="node-exporter",
          device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
        record: instance_device:node_disk_io_time_seconds:rate5m
      - expr:
          rate(node_disk_io_time_weighted_seconds_total{job="node-exporter",
          device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
        record: instance_device:node_disk_io_time_weighted_seconds:rate5m
      - expr: |-
          sum without (device) (
            rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m])
          )
        record: instance:node_network_receive_bytes_excluding_lo:rate5m
      - expr: |-
          sum without (device) (
            rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m])
          )
        record: instance:node_network_transmit_bytes_excluding_lo:rate5m
      - expr: |-
          sum without (device) (
            rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m])
          )
        record: instance:node_network_receive_drop_excluding_lo:rate5m
      - expr: |-
          sum without (device) (
            rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m])
          )
        record: instance:node_network_transmit_drop_excluding_lo:rate5m

  - name: node.rules
    rules:
      - expr: |-
          topk by(cluster, namespace, pod) (1,
            max by (cluster, node, namespace, pod) (
              label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")
          ))
        record: "node_namespace_pod:kube_pod_info:"
      - expr: |-
          count by (cluster, node) (
            node_cpu_seconds_total{mode="idle",job="node-exporter"}
            * on (namespace, pod) group_left(node)
            topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:)
          )
        record: node:node_num_cpu:sum
      - expr: |-
          sum(
            node_memory_MemAvailable_bytes{job="node-exporter"} or
            (
              node_memory_Buffers_bytes{job="node-exporter"} +
              node_memory_Cached_bytes{job="node-exporter"} +
              node_memory_MemFree_bytes{job="node-exporter"} +
              node_memory_Slab_bytes{job="node-exporter"}
            )
          ) by (cluster)
        record: :node_memory_MemAvailable_bytes:sum
      - expr: |-
          avg by (cluster, node) (
            sum without (mode) (
              rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m])
            )
          )
        record: node:node_cpu_utilization:ratio_rate5m
      - expr: |-
          avg by (cluster) (
            node:node_cpu_utilization:ratio_rate5m
          )
        record: cluster:node_cpu:ratio_rate5m