diff --git a/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-pods.json b/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-pods.json index 96419158..df0d4717 100644 --- a/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-pods.json +++ b/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-pods.json @@ -668,7 +668,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"})", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"})", "instant": true, "interval": "$resolution", "legendFormat": "Requests", @@ -738,7 +738,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"})", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"})", "instant": true, "interval": "$resolution", "legendFormat": "Limits", @@ -812,7 +812,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"})", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"})", "instant": true, "interval": "$resolution", "legendFormat": "Requests", @@ -882,7 +882,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) ", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) ", "instant": true, "interval": "$resolution", "legendFormat": "Limits", @@ -1057,7 +1057,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", "format": "table", "hide": false, "instant": true, @@ -1072,7 +1072,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", "format": "table", "hide": false, "instant": true, @@ -1274,7 +1274,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", "interval": "$resolution", "legendFormat": "{{ container }} REQUESTS", "range": true, @@ -1286,7 +1286,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", "hide": false, "legendFormat": "{{ container }} LIMITS", "range": true, @@ -1402,7 +1402,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", "interval": "", "legendFormat": "{{ container }} REQUESTS", "range": true, @@ -1414,7 +1414,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", cluster=\"$cluster\"}) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)", "hide": false, "legendFormat": "{{ container }} LIMITS", "range": true, @@ -1533,7 +1533,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", "interval": "$resolution", "legendFormat": "{{ container }}", "range": true, @@ -1635,7 +1635,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", "interval": "", "legendFormat": "{{ container }}", "range": true, @@ -1739,7 +1739,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\", pod=\"$pod\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", "interval": "$resolution", "legendFormat": "{{ container }}", "range": true, @@ -1867,7 +1867,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(increase(container_oom_events_total{namespace=\"${namespace}\", pod=\"${pod}\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "expr": "sum(increase(container_oom_events_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"${namespace}\", pod=\"${pod}\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", "interval": "", "legendFormat": "{{ container }}", "range": true, @@ -2104,7 +2104,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum(rate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Received", "refId": "A" @@ -2115,7 +2115,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "- sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "- sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Transmitted", "refId": "B" @@ -2210,7 +2210,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(rate(container_network_receive_packets_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum(rate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Received", "refId": "A" @@ -2221,7 +2221,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "- sum(rate(container_network_transmit_packets_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "- sum(rate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Transmitted", "refId": "B" @@ -2316,7 +2316,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(rate(container_network_receive_packets_dropped_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum(rate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Received", "refId": "A" @@ -2327,7 +2327,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "- sum(rate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "- sum(rate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Transmitted", "refId": "B" @@ -2422,7 +2422,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(rate(container_network_receive_errors_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum(rate(container_network_receive_errors_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Received", "refId": "A" @@ -2433,7 +2433,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "- sum(rate(container_network_transmit_errors_total{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "- sum(rate(container_network_transmit_errors_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "Transmitted", "refId": "B" diff --git a/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-service.json b/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-service.json new file mode 100644 index 00000000..e6d2e6c2 --- /dev/null +++ b/hosts/idols-aquamarine/grafana/dashboards/kubernetes/k8s-views-service.json @@ -0,0 +1,2645 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.3.4" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "This is a modern 'Pods View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 43, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Information", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_info{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ created_by_kind }}: {{ created_by_name }}", + "refId": "A" + } + ], + "title": "Created by", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "links": [ + { + "title": "", + "url": "/d/k8s_views_nodes/kubernetes-views-nodes?var-datasource=${datasource}&var-cluster=$cluster&var-node=${__data.fields.node}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 33, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_info{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ node }}", + "refId": "A" + } + ], + "title": "Running on", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 41, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_info{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ pod_ip }}", + "refId": "A" + } + ], + "title": "Pod IP", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 0, + "y": 3 + }, + "id": 52, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_info{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", priority_class!=\"\", cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{ priority_class }}", + "range": false, + "refId": "A" + } + ], + "title": "Priority Class", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Burstable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "BestEffort" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 2, + "w": 7, + "x": 5, + "y": 3 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_status_qos_class{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"} > 0", + "instant": true, + "interval": "", + "legendFormat": "{{ qos_class }}", + "refId": "A" + } + ], + "title": "QOS Class", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_container_status_last_terminated_reason{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ reason }}", + "refId": "A" + } + ], + "title": "Last Terminated Reason", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + }, + { + "color": "#EAB839", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 57, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_container_status_last_terminated_exitcode{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Last Terminated Exit Code", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 47, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 60 + }, + { + "color": "red", + "value": 75 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 39, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (pod) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (pod)", + "instant": true, + "interval": "$resolution", + "legendFormat": "Requests", + "refId": "A" + } + ], + "title": "Total pod CPU Requests usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 60 + }, + { + "color": "red", + "value": 75 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 48, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}[$__rate_interval])) by (pod) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (pod)", + "instant": true, + "interval": "$resolution", + "legendFormat": "Limits", + "refId": "A" + } + ], + "title": "Total pod CPU Limits usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "red", + "value": 99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 40, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) by (pod) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (pod)", + "instant": true, + "interval": "$resolution", + "legendFormat": "Requests", + "refId": "A" + } + ], + "title": "Total pod RAM Requests usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 60 + }, + { + "color": "red", + "value": 75 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 9, + "y": 6 + }, + "id": 49, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}) by (pod) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (pod) ", + "instant": true, + "interval": "$resolution", + "legendFormat": "Limits", + "refId": "B" + } + ], + "title": "Total pod RAM Limits usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false, + "minWidth": 100 + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Memory Requests" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Limits" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Used" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 38, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}) by (pod, container)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "F" + } + ], + "title": "Resources by container", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "container" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 4": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "container": false, + "endpoint": true, + "endpoint 2": true, + "endpoint 3": true, + "endpoint 4": true, + "instance": true, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "job": true, + "job 2": true, + "job 3": true, + "job 4": true, + "namespace": true, + "namespace 2": true, + "namespace 3": true, + "namespace 4": true, + "node": true, + "node 2": true, + "node 3": true, + "node 4": true, + "pod": true, + "pod 2": true, + "pod 3": true, + "pod 4": true, + "resource 1": true, + "resource 2": true, + "resource 3": true, + "resource 4": true, + "service": true, + "service 2": true, + "service 3": true, + "service 4": true, + "uid 1": true, + "uid 2": true, + "uid 3": true, + "uid 4": true, + "unit 1": true, + "unit 2": true, + "unit 3": true, + "unit 4": true + }, + "indexByName": { + "Time 1": 7, + "Time 2": 8, + "Time 3": 9, + "Time 4": 10, + "Time 5": 11, + "Time 6": 12, + "Value #A": 2, + "Value #B": 3, + "Value #C": 5, + "Value #D": 6, + "Value #E": 1, + "Value #F": 4, + "container": 0 + }, + "renameByName": { + "Value #A": "CPU Requests", + "Value #B": "CPU Limits", + "Value #C": "Memory Requests", + "Value #D": "Memory Limits", + "Value #E": "CPU Used", + "Value #F": "Memory Used", + "container": "Container" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Percent", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "green", + "value": 30 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - {{ container }} REQUESTS", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "hide": false, + "legendFormat": "{{ pod }} - {{ container }} LIMITS", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage / Requests & Limits by container", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Percent", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "green", + "value": 30 + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}) by (pod, container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "interval": "", + "legendFormat": "{{ pod }} - {{ container }} REQUESTS", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}) by (pod, container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (pod, container)", + "hide": false, + "legendFormat": "{{ pod }} - {{ container }} LIMITS", + "range": true, + "refId": "B" + } + ], + "title": "Memory Usage / Requests & Limits by container", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "CPU Cores", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 29, + "options": { + "legend": { + "calcs": ["min", "max", "mean"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - {{ container }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage by container", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 51, + "options": { + "legend": { + "calcs": ["min", "max", "mean"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}) by (pod, container)", + "interval": "", + "legendFormat": "{{ pod }} - {{ container }}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage by container", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "SECONDS", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 59, + "options": { + "legend": { + "calcs": ["min", "max", "mean"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container=\"$container\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - {{ container }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Throttled seconds by container", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 62, + "panels": [], + "title": "Kubernetes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Percent", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "green", + "value": 30 + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(container_oom_events_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"${namespace}\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container)", + "interval": "", + "legendFormat": "{{ pod }} - {{ container }}", + "range": true, + "refId": "A" + } + ], + "title": "OOM Events by container", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Percent", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "green", + "value": 30 + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"${namespace}\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container=\"$container\", job=~\"$job\", cluster=\"$cluster\"}[$__rate_interval])) by (pod, container)", + "interval": "", + "legendFormat": "{{ pod }} - {{ container }}", + "range": true, + "refId": "A" + } + ], + "title": "Container Restarts by container", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 45, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "- sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Transmitted", + "refId": "B" + } + ], + "title": "Network - Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "- sum(rate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Transmitted", + "refId": "B" + } + ], + "title": "Network - Packets Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "- sum(rate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Transmitted", + "refId": "B" + } + ], + "title": "Network - Packets Dropped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(container_network_receive_errors_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "- sum(rate(container_network_transmit_errors_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", cluster=\"$cluster\"}[$__rate_interval])) by (pod)", + "interval": "$resolution", + "legendFormat": "{{ pod }} - Transmitted", + "refId": "B" + } + ], + "title": "Network - Errors", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["Kubernetes", "Prometheus"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_node_info,cluster)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_node_info,cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_info{namespace=\"$namespace\", cluster=\"$cluster\"}, pod)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "service", + "options": [], + "query": { + "query": "label_values(kube_pod_info{namespace=\"$namespace\", cluster=\"$cluster\"}, pod)", + "refId": "Prometheus-service-Variable-Query" + }, + "refresh": 2, + "regex": "/^(.+)-[0-9a-f]{1,10}-[0-9-a-z]{5}$/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_cpu_usage_seconds_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}, container)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "container", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"$service-[0-9a-f]{1,10}-[0-9-a-z]{5}$\", container!=\"\", container!=\"POD\", cluster=\"$cluster\"}, container)", + "refId": "Prometheus-container-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "15s", + "value": "15s" + }, + { + "selected": true, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + } + ], + "query": "1s, 15s, 30s, 1m, 3m, 5m", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_info,job)", + "hide": 0, + "includeAll": false, + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes / Views / Service", + "uid": "k8s_views_services", + "version": 1, + "weekStart": "" +} diff --git a/hosts/idols-aquamarine/grafana/dashboards/kubernetes/kubernetes-persistent-volumes.json b/hosts/idols-aquamarine/grafana/dashboards/kubernetes/kubernetes-persistent-volumes.json new file mode 100644 index 00000000..44e407ae --- /dev/null +++ b/hosts/idols-aquamarine/grafana/dashboards/kubernetes/kubernetes-persistent-volumes.json @@ -0,0 +1,1623 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "description": "Monitors all persistent volumes that exists within a Kubernetes cluster. Prometheus is used as datasource. ", + "editable": true, + "gnetId": 13646, + "graphTooltip": 0, + "iteration": 1609275169694, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Alerts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + }, + { + "color": "light-green", + "value": -0.0001 + }, + { + "color": "semi-dark-orange", + "value": 0.0001 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n count (\n (kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"})\n and\n (predict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}[1d], 2 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PVCs Full in 2 days - Based on Daily Usage", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + }, + { + "color": "light-green", + "value": -0.0001 + }, + { + "color": "semi-dark-orange", + "value": 0.0001 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 28, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n count (\n (kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"})\n and\n (predict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}[1d], 5 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PVCs Full in 5 days - Based on Daily Usage", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + }, + { + "color": "light-green", + "value": -0.0001 + }, + { + "color": "semi-dark-orange", + "value": 0.0001 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 27, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n count (\n (kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"})\n and\n (predict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}[1d], 7 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PVCs Full in 1 Week - Based on Daily Usage", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "light-green", + "value": -0.0001 + }, + { + "color": "semi-dark-red", + "value": 0.0001 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} ) and (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} )) >= (${warning_threshold} / 100)) or vector (0)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PVCs Above Warning Threshold", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "light-green", + "value": -0.0001 + }, + { + "color": "semi-dark-red", + "value": 0.0001 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 24, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count((kube_persistentvolumeclaim_status_phase{cluster=\"$cluster\", namespace=~\"$namespace\",phase=\"Pending\"}==1)) or vector(0)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PVCs in Pending State", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "light-green", + "value": -0.0001 + }, + { + "color": "semi-dark-red", + "value": 0.0001 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 23, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count((kube_persistentvolumeclaim_status_phase{cluster=\"$cluster\", namespace=~\"$namespace\",phase=\"Lost\"}==1)) or vector(0)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PVCs in Lost State", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 17, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Usage statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "cellOptions": { + "type": "auto" + }, + "filterable": false + }, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used (%)" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + }, + { + "color": "semi-dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 80 + } + ] + } + }, + { + "id": "decimals", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "text": "Bound" + }, + "1": { + "text": "Pending" + }, + "2": { + "text": "Lost" + } + }, + "type": "value" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + }, + { + "color": "light-green", + "value": 0 + }, + { + "color": "semi-dark-orange", + "value": 1 + }, + { + "color": "semi-dark-red", + "value": 2 + } + ] + } + }, + { + "id": "noValue", + "value": "--" + }, + { + "id": "custom.align", + "value": "center" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Capacity (GiB)" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used (GiB)" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available (GiB)" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "StorageClass" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PersistentVolumeClaim" + }, + "properties": [ + { + "id": "custom.width", + "value": 370 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 29, + "interval": "", + "options": { + "frameIndex": 2, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "PersistentVolumeClaim" + } + ] + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "label_join(sum by (persistentvolumeclaim, namespace, storageclass, volumename) (kube_persistentvolumeclaim_info{cluster=\"$cluster\", namespace=~\"$namespace\"}), \"pvc_key\", \"/\", \"namespace\", \"persistentvolumeclaim\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pvc_key) (label_join(sum by (persistentvolumeclaim, namespace) (kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} / 1024 / 1024 / 1024), \"pvc_key\", \"/\", \"namespace\", \"persistentvolumeclaim\"))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pvc_key) (label_join(sum by (persistentvolumeclaim, namespace) (kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} / 1024 / 1024 / 1024), \"pvc_key\", \"/\", \"namespace\", \"persistentvolumeclaim\"))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pvc_key) (label_join(sum by (persistentvolumeclaim, namespace) (kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} / 1024 / 1024 / 1024), \"pvc_key\", \"/\", \"namespace\", \"persistentvolumeclaim\"))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pvc_key) (label_join(sum by (persistentvolumeclaim, namespace) (kube_persistentvolumeclaim_status_phase{cluster=\"$cluster\", namespace=~\"$namespace\", phase=~\"(Pending|Lost)\"}) + sum by (persistentvolumeclaim, namespace) (kube_persistentvolumeclaim_status_phase{cluster=\"$cluster\", namespace=~\"$namespace\", phase=\"Lost\"}), \"pvc_key\", \"/\", \"namespace\", \"persistentvolumeclaim\"))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pvc_key) (label_join(sum by (persistentvolumeclaim, namespace) (kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"} * 100), \"pvc_key\", \"/\", \"namespace\", \"persistentvolumeclaim\"))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Persistent Volume Claim", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "pvc_key" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Value #A": true, + "pvc_key": true + }, + "indexByName": {}, + "renameByName": { + "Time 1": "", + "Time 2": "", + "Time 3": "", + "Time 4": "", + "Time 5": "", + "Time 6": "", + "Value #A": "", + "Value #B": "Capacity (GiB)", + "Value #C": "Used (GiB)", + "Value #D": "Available (GiB)", + "Value #E": "Status", + "Value #F": "Used (%)", + "namespace": "Namespace", + "persistentvolumeclaim": "PersistentVolumeClaim", + "storageclass": "StorageClass", + "volumename": "PhysicalVolume" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 7, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Status" + } + ] + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "kube_storageclass_info{cluster=\"$cluster\"}", + "format": "table", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Class", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "app_kubernetes_io_instance": true, + "app_kubernetes_io_name": true, + "instance": true, + "job": true, + "kubernetes_namespace": true, + "kubernetes_pod_name": true, + "pod_template_hash": true + }, + "indexByName": { + "Time": 1, + "Value": 13, + "__name__": 2, + "app_kubernetes_io_instance": 3, + "app_kubernetes_io_name": 4, + "instance": 5, + "job": 6, + "kubernetes_namespace": 7, + "kubernetes_pod_name": 8, + "pod_template_hash": 9, + "provisioner": 10, + "reclaimPolicy": 11, + "storageclass": 0, + "volumeBindingMode": 12 + }, + "renameByName": { + "provisioner": "Provisioner", + "reclaimPolicy": "ReclaimPolicy", + "storageclass": "StorageClass", + "volumeBindingMode": "VolumeBindingMode" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Provisioner": { + "aggregations": [], + "operation": "groupby" + }, + "ReclaimPolicy": { + "aggregations": [], + "operation": "groupby" + }, + "StorageClass": { + "aggregations": [], + "operation": "groupby" + }, + "VolumeBindingMode": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 15, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Graphical usage data ", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}))", + "interval": "", + "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "All Running PVCs Used Bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Date & time", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 19, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}[1h])", + "instant": false, + "interval": "", + "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hourly Volume Usage Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "binBps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Date & time", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}[1d])", + "interval": "", + "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Daily Volume Usage Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "binBps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Date & time", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(kubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\", cluster=\"$cluster\", namespace=~\"$namespace\"}[1w])", + "instant": false, + "interval": "", + "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Weekly Volume Usage Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "binBps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Date & time", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Usage Rate", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 42, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_node_info, cluster)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_node_info, cluster)", + "refId": "PrometheusVariableQueryEditor-Cluster" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_persistentvolumeclaim_info{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_persistentvolumeclaim_info{cluster=\"$cluster\"}, namespace)", + "refId": "PrometheusVariableQueryEditor-Namespace" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "80", + "value": "80" + }, + "hide": 2, + "label": "PVC % Used Warning Threshold", + "name": "warning_threshold", + "options": [ + { + "selected": true, + "text": "80", + "value": "80" + } + ], + "query": "80", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes / Storage / Persistent Volumes", + "uid": "k8s_storage_persistent_volumes", + "version": 1, + "id": null +} diff --git a/hosts/idols-aquamarine/monitoring/recoding_rules/k8s.yml b/hosts/idols-aquamarine/monitoring/recoding_rules/k8s.yml index ca3d4a4a..4f169548 100644 --- a/hosts/idols-aquamarine/monitoring/recoding_rules/k8s.yml +++ b/hosts/idols-aquamarine/monitoring/recoding_rules/k8s.yml @@ -3,31 +3,31 @@ groups: rules: - expr: |- sum by (cluster, namespace, pod, container) ( - irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", container!="", container!="POD"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate - expr: |- - container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", container!="", container!="POD"} * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_working_set_bytes - expr: |- - container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", container!="", container!="POD"} * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_rss - expr: |- - container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", container!="", container!="POD"} * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_cache - expr: |- - container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", container!="", container!="POD"} * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) )