{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "", "editable": true, "gnetId": null, "graphTooltip": 0, "id": 4, "iteration": 1643881023858, "links": [], "panels": [ { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 65, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 2, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 20, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_from_creation_seconds_bucket{instance=~\"$instance\"}[5m])) by (phase, le))", "hide": false, "interval": "", "legendFormat": "{{phase}}", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [ { "colorMode": "background6", "fill": true, "fillColor": "rgba(234, 112, 112, 0.12)", "line": false, "lineColor": "rgba(237, 46, 24, 0.60)", "op": "time" } ], "timeShift": null, "title": "VMI Creation Time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "hiddenSeries": false, "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(kubevirt_vmi_phase_transition_time_from_creation_seconds_count{phase=\"Running\", instance=~\"$instance\"}[5m])) by (instance)", "interval": "", "legendFormat": "{{instance}} Running", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [ { "colorMode": "background6", "fill": true, "fillColor": "rgba(234, 112, 112, 0.12)", "line": false, "lineColor": "rgba(237, 46, 24, 0.60)", "op": "time" } ], "timeShift": null, "title": "VMI Start Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "decimals": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 22, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_seconds_bucket{phase=\"Pending\", instance=~\"$instance\"}[5m])) by (le,phase))", "interval": "", "legendFormat": "{{phase}}", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_seconds_bucket{phase=\"Scheduling\", instance=~\"$instance\"}[5m])) by (le,phase))", "hide": false, "interval": "", "legendFormat": "{{phase}}", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_seconds_bucket{phase=\"Scheduled\", instance=~\"$instance\"}[5m])) by (le,phase))", "hide": false, "interval": "", "legendFormat": "{{phase}}", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_seconds_bucket{phase=\"Running\", instance=~\"$instance\"}[5m])) by (le,phase))", "hide": false, "interval": "", "legendFormat": "{{phase}}", "refId": "D" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_seconds_bucket{phase=\"Succeeded\", instance=~\"$instance\"}[5m])) by (le,phase))", "hide": false, "interval": "", "legendFormat": "{{phase}}", "refId": "E" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(kubevirt_vmi_phase_transition_time_seconds_bucket{phase=\"Failed\", instance=~\"$instance\"}[5m])) by (le,phase))", "hide": false, "interval": "", "legendFormat": "{{phase}}", "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VMI Phase Transition Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "hiddenSeries": false, "id": 41, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(increase(kubevirt_vmi_phase_transition_time_from_creation_seconds_count{phase=\"Failed\", instance=~\"$instance\"}[20m])) by (instance)", "interval": "", "legendFormat": "{{instance}} Failed", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, "expr": "sum(increase(kubevirt_vmi_phase_transition_time_from_creation_seconds_count{phase=\"Running\", instance=~\"$instance\"}[20m])) by (instance)", "hide": false, "interval": "", "legendFormat": "{{instance}} Running", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [ { "colorMode": "background6", "fill": true, "fillColor": "rgba(234, 112, 112, 0.12)", "line": false, "lineColor": "rgba(237, 46, 24, 0.60)", "op": "time" } ], "timeShift": null, "title": "VMI Count (approx.)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Virtual Machines", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 52, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many read requests (LIST,GET,WATCH) per second do the apiservers get by code?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 9, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ { "alias": "/2../i", "color": "#56A64B" }, { "alias": "/3../i", "color": "#F2CC0C" }, { "alias": "/4.*kubevirt/i", "color": "#3274D9" }, { "alias": "/5../i", "color": "#E02F44" }, { "alias": "/4.*cdi/i", "color": "#8F3BB8" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_total{group=~\".*kubevirt.*\", instance=~\"$instance\", verb=~\"LIST|GET\"}[5m])) by (code, group)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{ code }} {{group}} ", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server - Read Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many seconds is the 99th percentile for reading (LIST|WATCH|GET) a given resource?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 2 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{group=\"kubevirt.io\", instance=~\"$instance\", verb=~\"LIST|GET\", scope=\"cluster\"}[5m])) by (le, resource, verb))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{verb}} {{ resource }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server - Read Requests Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 10, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ { "alias": "/2../i", "color": "#56A64B" }, { "alias": "/3../i", "color": "#F2CC0C" }, { "alias": "/4../i", "color": "#3274D9" }, { "alias": "/5../i", "color": "#E02F44" }, { "alias": "/409/i", "color": "#C0D8FF" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(irate(rest_client_requests_total{instance=~\"$instance\", pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\", container!=\"\", method=~\"POST|PUT|PATCH|DELETE\"}[2m])) by (code, group, container, job, verb, method) > 0", "interval": "", "legendFormat": "{{container}} - {{method}} - {{code}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server - Write Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 4, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{group=\"kubevirt.io\", instance=~\"$instance\", verb=~\"POST|PUT|PATCH|DELETE\"}[5m])) by (le, verb, resource)) > 0", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ resource }} - {{verb}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server - Write Requests Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 47, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/2../i", "color": "#56A64B" }, { "alias": "/3../i", "color": "#F2CC0C" }, { "alias": "/4../i", "color": "#3274D9" }, { "alias": "/5../i", "color": "#E02F44" }, { "alias": "/409/i", "color": "#C0D8FF" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(irate(rest_client_requests_total{instance=~\"$instance\", pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\", container!=\"\", method=~\"GET|LIST\"}[2m])) by (code, container, job, verb, method) > 0", "interval": "", "legendFormat": "{{container}} - {{method}} - {{code}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Read Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 4, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 62, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{instance=~\"$instance\", verb=~\"GET|LIST\", container!=\"\"}[2m])) by (le, verb, container)) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{container}} - {{verb}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Read Requests Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 48, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/2../i", "color": "#56A64B" }, { "alias": "/3../i", "color": "#F2CC0C" }, { "alias": "/4../i", "color": "#3274D9" }, { "alias": "/5../i", "color": "#E02F44" }, { "alias": "/409/i", "color": "#C0D8FF" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(irate(rest_client_requests_total{instance=~\"$instance\", pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\", container!=\"\", method=~\"POST|PUT|PATCH|DELETE\"}[2m])) by (code, container, job, verb, method) > 0", "hide": false, "interval": "", "legendFormat": "{{container}} - {{method}} - {{code}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Write Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 4, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 63, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{instance=~\"$instance\", verb=~\"POST|PUT|PATCH|DELETE\", container!=\"\"}[2m])) by (le, verb, container)) > 0", "interval": "", "legendFormat": "{{container}} - {{verb}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Write Requests Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 37, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(irate(rest_client_rate_limiter_duration_seconds_bucket{instance=~\"$instance\", container!=\"\"}[5m])) by (container, le))", "interval": "", "legendFormat": "{{container}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Rate Limiter Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "KubeVirt API Access", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 54, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 7, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 13, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(kubevirt_workqueue_adds_total{job=~\".*kubevirt.*\", instance=~\"$instance\"}[1m])) by (instance, name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Add Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ops", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 14, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "kubevirt_workqueue_depth{job=~\".*kubevirt.*\", instance=~\"$instance\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}} ", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Depth", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "cps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 7, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 15, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(kubevirt_workqueue_queue_duration_seconds_bucket{job=~\".*kubevirt.*\", instance=~\"$instance\"}[1m])) by (instance, name, le))", "hide": false, "interval": "", "legendFormat": "{{name}} {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Queue Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 7, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 46, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(kubevirt_workqueue_work_duration_seconds_bucket{job=~\".*kubevirt.*\", instance=~\"$instance\"}[1m])) by (instance, name, le))", "hide": false, "interval": "", "legendFormat": "{{name}} {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Work Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 7, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 24, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "kubevirt_workqueue_unfinished_work_seconds{job=~\".*kubevirt.*\", instance=~\"$instance\"}", "interval": "", "legendFormat": "{{name}} {{instance}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Unfinished Work", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 }, "hiddenSeries": false, "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(kubevirt_workqueue_retries_total{instance=~\"$instance\"}[1m])", "interval": "", "legendFormat": "{{name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Retry Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "How many seconds has the longest running processor for workqueue been running.", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 8, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 35 }, "hiddenSeries": false, "id": 26, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "kubevirt_workqueue_longest_running_processor_seconds{job=~\".*kubevirt.*\", instance=~\"$instance\"}", "interval": "", "legendFormat": "{{name}} {{instance}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Work Queue - Longest Running Processor", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "KubeVirt Work Queue", "type": "row" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, "id": 50, "panels": [], "title": "K8s API Access", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "hiddenSeries": false, "id": 66, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/2../i", "color": "#56A64B" }, { "alias": "/3../i", "color": "#F2CC0C" }, { "alias": "/4../i", "color": "#3274D9" }, { "alias": "/5../i", "color": "#E02F44" }, { "alias": "/409/i", "color": "#C0D8FF" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(irate(rest_client_requests_total{instance=~\"$instance\", pod!~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\", container!=\"\", method=~\"GET|LIST\"}[2m])) by (code, container, job, verb, method) > 0", "interval": "", "legendFormat": "{{container}} - {{method}} - {{code}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Read Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many seconds is the 99th percentile for writing (GET|LIST\") a given resource?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 4, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "hiddenSeries": false, "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{group!=\"kubevirt.io\", instance=~\"$instance\", verb=~\"GET|LIST\"}[5m])) by (le, verb, resource)) > 0", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{verb}} {{ resource }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server - Read Requests Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 68, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/2../i", "color": "#56A64B" }, { "alias": "/3../i", "color": "#F2CC0C" }, { "alias": "/4../i", "color": "#3274D9" }, { "alias": "/5../i", "color": "#E02F44" }, { "alias": "/409/i", "color": "#C0D8FF" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(irate(rest_client_requests_total{instance=~\"$instance\", pod!~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\", container!=\"\", method=~\"POST|PUT|PATCH|DELETE\"}[2m])) by (code, container, job, verb, method) > 0", "interval": "", "legendFormat": "{{container}} - {{method}} - {{code}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Client - Write Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 4, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 69, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{group!=\"kubevirt.io\", instance=~\"$instance\", verb=~\"POST|PUT|PATCH|DELETE\"}[5m])) by (le, verb, resource)) > 0", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{verb}} {{ resource }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server - Write Requests Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "hiddenSeries": false, "id": 70, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{}[5m])) by (le, container))", "interval": "", "legendFormat": "{{container}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Schedule Latency", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 10, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "hiddenSeries": false, "id": 71, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": false }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{}[5m])) by (instance, le)) > 0", "interval": "", "legendFormat": "{{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Start Latency", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 58, "panels": [], "title": "Resource Usage", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 12, "w": 6, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 16, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(process_resident_memory_bytes{job=~\".*kubevirt.*\", instance=~\"$instance\"}) by (instance, container)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{container}} {{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 12, "w": 6, "x": 6, "y": 29 }, "hiddenSeries": false, "id": 17, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(process_cpu_seconds_total{job=~\".*kubevirt.*\", instance=~\"$instance\"}[5m])) by (instance, container)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{container}} {{instance}} ", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU usage", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 0, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 12, "w": 6, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 18, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(process_open_fds{container!=\"\"}[10m])) by (container)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{container}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Open Files", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 12, "w": 6, "x": 18, "y": 29 }, "hiddenSeries": false, "id": 28, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(container_network_receive_bytes_total{namespace=~\".*kubevirt.*|openshift-cnv\", instance=~\"$instance\"}[5m])) by (node, pod) +\nsum(rate(container_network_transmit_bytes_total{namespace=~\".*kubevirt.*|openshift-cnv\", instance=~\"$instance\"}[5m])) by (node, pod)", "interval": "", "legendFormat": "{{pod}} {{node}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 2, "fillGradient": 0, "gridPos": { "h": 12, "w": 8, "x": 0, "y": 41 }, "hiddenSeries": false, "id": 30, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(go_gc_duration_seconds_sum{}[10m])) by (container) / sum(rate(go_gc_duration_seconds_count{}[10m])) by (container)", "interval": "", "legendFormat": "{{container}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "GC Duration Mean", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 0, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 12, "w": 8, "x": 8, "y": 41 }, "hiddenSeries": false, "id": 44, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_goroutines{job=~\".*kubevirt.*\", instance=~\"$instance\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{container}} {{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Goroutines", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 0, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 12, "w": 8, "x": 16, "y": 41 }, "hiddenSeries": false, "id": 45, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(go_threads{container!=\"\"}) by(container)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{container}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Threads", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 53 }, "id": 56, "panels": [], "title": "Storage Operations", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 54 }, "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(storage_operation_duration_seconds_count{metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", "interval": "", "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Storage Operation Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 54 }, "hiddenSeries": false, "id": 40, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(storage_operation_errors_total{metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", "interval": "", "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Storage Operation Error Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 64 }, "id": 60, "panels": [], "title": "etcd", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 65 }, "hiddenSeries": false, "id": 42, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(grpc_client_handled_total{grpc_type=\"unary\", instance=~\"$instance\"}[5m]))", "interval": "", "legendFormat": "RPC Rate", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(grpc_client_handled_total{grpc_type=\"unary\", grpc_code!=\"OK\", instance=~\"$instance\"}[5m]))", "hide": false, "interval": "", "legendFormat": "RCP Failed Rate", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd RPC Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 65 }, "hiddenSeries": false, "id": 32, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le,operation,apiserver)) > 0", "interval": "", "legendFormat": "{{apiserver}} - {{operation}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99th %ile all Etcd Request Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 65 }, "hiddenSeries": false, "id": 33, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(etcd_db_total_size_in_bytes{instance=~\"$instance\"}) by (instance)", "interval": "", "legendFormat": "{{instance}} ", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd DB Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 73 }, "hiddenSeries": false, "id": 73, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{instance=~\"$instance\"}[2m])) > 0", "interval": "", "legendFormat": "{{instance}} ", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99th %ile all Etcd Wal Fsync Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 73 }, "hiddenSeries": false, "id": 74, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{instance=~\"$instance\"}[2m])) > 0", "interval": "", "legendFormat": "{{instance}} ", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99th %ile all Etcd RTT Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 73 }, "hiddenSeries": false, "id": 75, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(etcd_object_counts{instance=~\"$instance\"}[5m])) by (resource) > 0", "interval": "", "legendFormat": "{{resource}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd Resource Operation Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "1m", "schemaVersion": 26, "style": "dark", "tags": ["kubevirt", "kubevirt-control-plane", "sig-scale"], "templating": { "list": [ { "current": { "selected": false, "text": "Cluster Prometheus", "value": "Cluster Prometheus" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": { "isNone": true, "selected": false, "text": "None", "value": "" }, "datasource": "$datasource", "definition": "label_values(apiserver_request_total, cluster)", "error": null, "hide": 2, "includeAll": false, "label": "cluster", "multi": false, "name": "cluster", "options": [ { "isNone": true, "selected": true, "text": "None", "value": "" } ], "query": "label_values(apiserver_request_total, cluster)", "refresh": 0, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": "$datasource", "definition": "label_values(container_cpu_usage_seconds_total{service=\"kubelet\", cluster=\"$cluster\"}, instance)", "error": null, "hide": 0, "includeAll": true, "label": "instance", "multi": true, "name": "instance", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": "192.168.216.11:10250", "value": "192.168.216.11:10250" }, { "selected": false, "text": "192.168.216.14:10250", "value": "192.168.216.14:10250" }, { "selected": false, "text": "192.168.216.61:10250", "value": "192.168.216.61:10250" }, { "selected": false, "text": "192.168.216.48:10250", "value": "192.168.216.48:10250" }, { "selected": false, "text": "192.168.216.12:10250", "value": "192.168.216.12:10250" }, { "selected": false, "text": "192.168.216.44:10250", "value": "192.168.216.44:10250" }, { "selected": false, "text": "192.168.216.90:10250", "value": "192.168.216.90:10250" }, { "selected": false, "text": "192.168.216.94:10250", "value": "192.168.216.94:10250" }, { "selected": false, "text": "192.168.216.101:10250", "value": "192.168.216.101:10250" }, { "selected": false, "text": "192.168.216.49:10250", "value": "192.168.216.49:10250" }, { "selected": false, "text": "192.168.216.74:10250", "value": "192.168.216.74:10250" }, { "selected": false, "text": "192.168.216.15:10250", "value": "192.168.216.15:10250" }, { "selected": false, "text": "192.168.216.80:10250", "value": "192.168.216.80:10250" }, { "selected": false, "text": "192.168.216.95:10250", "value": "192.168.216.95:10250" }, { "selected": false, "text": "192.168.216.60:10250", "value": "192.168.216.60:10250" }, { "selected": false, "text": "192.168.216.76:10250", "value": "192.168.216.76:10250" }, { "selected": false, "text": "192.168.216.62:10250", "value": "192.168.216.62:10250" }, { "selected": false, "text": "192.168.216.18:10250", "value": "192.168.216.18:10250" }, { "selected": false, "text": "192.168.216.99:10250", "value": "192.168.216.99:10250" }, { "selected": false, "text": "192.168.216.86:10250", "value": "192.168.216.86:10250" }, { "selected": false, "text": "192.168.216.17:10250", "value": "192.168.216.17:10250" }, { "selected": false, "text": "192.168.216.47:10250", "value": "192.168.216.47:10250" }, { "selected": false, "text": "192.168.216.73:10250", "value": "192.168.216.73:10250" }, { "selected": false, "text": "192.168.216.91:10250", "value": "192.168.216.91:10250" }, { "selected": false, "text": "192.168.216.68:10250", "value": "192.168.216.68:10250" }, { "selected": false, "text": "192.168.216.10:10250", "value": "192.168.216.10:10250" }, { "selected": false, "text": "192.168.216.82:10250", "value": "192.168.216.82:10250" }, { "selected": false, "text": "192.168.216.31:10250", "value": "192.168.216.31:10250" }, { "selected": false, "text": "192.168.216.33:10250", "value": "192.168.216.33:10250" }, { "selected": false, "text": "192.168.216.67:10250", "value": "192.168.216.67:10250" }, { "selected": false, "text": "192.168.216.56:10250", "value": "192.168.216.56:10250" }, { "selected": false, "text": "192.168.216.34:10250", "value": "192.168.216.34:10250" }, { "selected": false, "text": "192.168.216.89:10250", "value": "192.168.216.89:10250" }, { "selected": false, "text": "192.168.216.27:10250", "value": "192.168.216.27:10250" }, { "selected": false, "text": "192.168.216.77:10250", "value": "192.168.216.77:10250" }, { "selected": false, "text": "192.168.216.24:10250", "value": "192.168.216.24:10250" }, { "selected": false, "text": "192.168.216.52:10250", "value": "192.168.216.52:10250" }, { "selected": false, "text": "192.168.216.58:10250", "value": "192.168.216.58:10250" }, { "selected": false, "text": "192.168.216.30:10250", "value": "192.168.216.30:10250" }, { "selected": false, "text": "192.168.216.102:10250", "value": "192.168.216.102:10250" }, { "selected": false, "text": "192.168.216.72:10250", "value": "192.168.216.72:10250" }, { "selected": false, "text": "192.168.216.78:10250", "value": "192.168.216.78:10250" }, { "selected": false, "text": "192.168.216.83:10250", "value": "192.168.216.83:10250" }, { "selected": false, "text": "192.168.216.79:10250", "value": "192.168.216.79:10250" }, { "selected": false, "text": "192.168.216.65:10250", "value": "192.168.216.65:10250" }, { "selected": false, "text": "192.168.216.45:10250", "value": "192.168.216.45:10250" }, { "selected": false, "text": "192.168.216.70:10250", "value": "192.168.216.70:10250" }, { "selected": false, "text": "192.168.216.98:10250", "value": "192.168.216.98:10250" }, { "selected": false, "text": "192.168.216.23:10250", "value": "192.168.216.23:10250" }, { "selected": false, "text": "192.168.216.37:10250", "value": "192.168.216.37:10250" }, { "selected": false, "text": "192.168.216.92:10250", "value": "192.168.216.92:10250" }, { "selected": false, "text": "192.168.216.107:10250", "value": "192.168.216.107:10250" }, { "selected": false, "text": "192.168.216.51:10250", "value": "192.168.216.51:10250" }, { "selected": false, "text": "192.168.216.41:10250", "value": "192.168.216.41:10250" }, { "selected": false, "text": "192.168.216.13:10250", "value": "192.168.216.13:10250" }, { "selected": false, "text": "192.168.216.19:10250", "value": "192.168.216.19:10250" }, { "selected": false, "text": "192.168.216.53:10250", "value": "192.168.216.53:10250" }, { "selected": false, "text": "192.168.216.39:10250", "value": "192.168.216.39:10250" }, { "selected": false, "text": "192.168.216.87:10250", "value": "192.168.216.87:10250" }, { "selected": false, "text": "192.168.216.38:10250", "value": "192.168.216.38:10250" }, { "selected": false, "text": "192.168.216.29:10250", "value": "192.168.216.29:10250" }, { "selected": false, "text": "192.168.216.71:10250", "value": "192.168.216.71:10250" }, { "selected": false, "text": "192.168.216.66:10250", "value": "192.168.216.66:10250" }, { "selected": false, "text": "192.168.216.100:10250", "value": "192.168.216.100:10250" }, { "selected": false, "text": "192.168.216.97:10250", "value": "192.168.216.97:10250" }, { "selected": false, "text": "192.168.216.54:10250", "value": "192.168.216.54:10250" }, { "selected": false, "text": "192.168.216.88:10250", "value": "192.168.216.88:10250" }, { "selected": false, "text": "192.168.216.69:10250", "value": "192.168.216.69:10250" }, { "selected": false, "text": "192.168.216.25:10250", "value": "192.168.216.25:10250" }, { "selected": false, "text": "192.168.216.84:10250", "value": "192.168.216.84:10250" }, { "selected": false, "text": "192.168.216.40:10250", "value": "192.168.216.40:10250" }, { "selected": false, "text": "192.168.216.57:10250", "value": "192.168.216.57:10250" }, { "selected": false, "text": "192.168.216.104:10250", "value": "192.168.216.104:10250" }, { "selected": false, "text": "192.168.216.35:10250", "value": "192.168.216.35:10250" }, { "selected": false, "text": "192.168.216.81:10250", "value": "192.168.216.81:10250" }, { "selected": false, "text": "192.168.216.93:10250", "value": "192.168.216.93:10250" }, { "selected": false, "text": "192.168.216.16:10250", "value": "192.168.216.16:10250" }, { "selected": false, "text": "192.168.216.63:10250", "value": "192.168.216.63:10250" }, { "selected": false, "text": "192.168.216.28:10250", "value": "192.168.216.28:10250" }, { "selected": false, "text": "192.168.216.21:10250", "value": "192.168.216.21:10250" }, { "selected": false, "text": "192.168.216.106:10250", "value": "192.168.216.106:10250" }, { "selected": false, "text": "192.168.216.103:10250", "value": "192.168.216.103:10250" }, { "selected": false, "text": "192.168.216.75:10250", "value": "192.168.216.75:10250" }, { "selected": false, "text": "192.168.216.55:10250", "value": "192.168.216.55:10250" }, { "selected": false, "text": "192.168.216.32:10250", "value": "192.168.216.32:10250" }, { "selected": false, "text": "192.168.216.36:10250", "value": "192.168.216.36:10250" }, { "selected": false, "text": "192.168.216.20:10250", "value": "192.168.216.20:10250" }, { "selected": false, "text": "192.168.216.50:10250", "value": "192.168.216.50:10250" }, { "selected": false, "text": "192.168.216.59:10250", "value": "192.168.216.59:10250" }, { "selected": false, "text": "192.168.216.64:10250", "value": "192.168.216.64:10250" }, { "selected": false, "text": "192.168.216.85:10250", "value": "192.168.216.85:10250" }, { "selected": false, "text": "192.168.216.46:10250", "value": "192.168.216.46:10250" }, { "selected": false, "text": "192.168.216.96:10250", "value": "192.168.216.96:10250" }, { "selected": false, "text": "192.168.216.26:10250", "value": "192.168.216.26:10250" }, { "selected": false, "text": "192.168.216.42:10250", "value": "192.168.216.42:10250" }, { "selected": false, "text": "192.168.216.43:10250", "value": "192.168.216.43:10250" }, { "selected": false, "text": "192.168.216.105:10250", "value": "192.168.216.105:10250" }, { "selected": false, "text": "192.168.216.108:10250", "value": "192.168.216.108:10250" } ], "query": "label_values(container_cpu_usage_seconds_total{service=\"kubelet\", cluster=\"$cluster\"}, instance)", "refresh": 0, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "UTC", "title": "KubeVirt / Control Plane", "uid": "kubevirt-control-plane", "version": 3 }