{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "This dashboard contains SCONE metrics.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 0, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 33, "panels": [], "repeat": "cas", "title": "CAS metrics", "type": "row" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 3, "w": 24, "x": 0, "y": 1 }, "id": 34, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "# CAS metrics\n\nThis section contains metrics pertaining to the CAS.", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 4 }, "id": 40, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Attestability", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"attestable\".\n\nStatus field description: The attestable state of the CAS, i.e., whether the SCONE operator could attest the CAS.\n\nThe value Yes indicates that the CAS was successfully attested.\n\nThe value No indicates that the CAS has not yet been provisioned.\n\nThe value Failed indicates that the attestation of the CAS failed.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"Failed\". The value 0 indicates the status field value \"No\". The value 1 indicates the status field value \"Yes\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 1, "text": "No" }, "1": { "color": "green", "index": 0, "text": "Yes" }, "-1": { "color": "#808080", "index": 2, "text": "Failed" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 6 }, "id": 39, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_attestable{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CAS attestable", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"attestable\".\n\nStatus field description: The attestable state of the CAS, i.e., whether the SCONE operator could attest the CAS.\n\nThe value Yes indicates that the CAS was successfully attested.\n\nThe value No indicates that the CAS has not yet been provisioned.\n\nThe value Failed indicates that the attestation of the CAS failed.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"Failed\". The value 0 indicates the status field value \"No\". The value 1 indicates the status field value \"Yes\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "red", "index": 1, "text": "No" }, "1": { "color": "green", "index": 0, "text": "Yes" }, "-1": { "color": "#808080", "index": 2, "text": "Failed" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "red", "value": 0 }, { "color": "green", "value": 1 } ] } }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "scone_cas_attestable" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 6 }, "id": 41, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_attestable{cr_name=\"$cas_instance\", cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "CAS attestable", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 11 }, "id": 38, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## State", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "uid": "prometheus" }, "description": "This metric corresponds to the status field \"state\".\n\nStatus field description: The state of the CAS.\n\nThe value HEALTHY indicates the CAS is running without any detected issues.\n\nThe value PENDING indicates the CAS is not yet healthy but no problems have been detected, and it is expected to become healthy without human intervention.\n\nThe value UNHEALTHY indicates a problem has been detected.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"UNHEALTHY\". The value 0 indicates the status field value \"PENDING\". The value 1 indicates the status field value \"HEALTHY\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "yellow", "index": 1, "text": "PENDING" }, "1": { "color": "green", "index": 2, "text": "HEALTHY" }, "-1": { "color": "red", "index": 0, "text": "UNHEALTHY" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "scone_sgxplugin_state" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 13 }, "id": 36, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_cas_state{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "CAS state", "type": "stat" }, { "datasource": { "uid": "prometheus" }, "description": "This metric corresponds to the status field \"state\".\n\nStatus field description: The state of the CAS.\n\nThe value HEALTHY indicates the CAS is running without any detected issues.\n\nThe value PENDING indicates the CAS is not yet healthy but no problems have been detected, and it is expected to become healthy without human intervention.\n\nThe value UNHEALTHY indicates a problem has been detected.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"UNHEALTHY\". The value 0 indicates the status field value \"PENDING\". The value 1 indicates the status field value \"HEALTHY\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds", "seriesBy": "last" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "yellow", "index": 1, "text": "PENDING" }, "1": { "color": "green", "index": 2, "text": "HEALTHY" }, "-1": { "color": "red", "index": 0, "text": "UNHEALTHY" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": -1 }, { "color": "#EAB839", "value": 0 }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 13 }, "id": 37, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_cas_state{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "CAS state", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 18 }, "id": 42, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Provisioned", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"provisioned\".\n\nStatus field description: The provisioned state of the CAS, i.e., whether the CAS is provisioned.\n\nThe value Yes indicates that the CAS is provisioned.\n\nThe value No indicates that the CAS is not provisioned.\n\n(Limitation in current implementation: Whether the CAS has been provisioned is currently determined by trying to retrieve a policy called \"provisioned\" from the CAS. On failure, the CAS is assumed to not have been provisioned. This might not be correct, in the case when the cas was indeed provisioned, but not by using the \"kubectl provision cas\" command. Using the command \"kubectl provision cas\" to provision a CAS, always creates a CAS policy called \"provisioned\".\n\nPossible metrics values: -1, 0, 1. The value -1 indicates that the metric could not be calculated. The value 0 indicates the status field value \"No\". The value 1 indicates the status field value \"Yes\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 1, "text": "No" }, "1": { "color": "green", "index": 0, "text": "Yes" }, "-1": { "color": "#808080", "index": 2, "text": "Unable to calculate" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 20 }, "id": 46, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_provisioned{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CAS provisioned", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"provisioned\".\n\nStatus field description: The provisioned state of the CAS, i.e., whether the CAS is provisioned.\n\nThe value Yes indicates that the CAS is provisioned.\n\nThe value No indicates that the CAS is not provisioned.\n\n(Limitation in current implementation: Whether the CAS has been provisioned is currently determined by trying to retrieve a policy called \"provisioned\" from the CAS. On failure, the CAS is assumed to not have been provisioned. This might not be correct, in the case when the cas was indeed provisioned, but not by using the \"kubectl provision cas\" command. Using the command \"kubectl provision cas\" to provision a CAS, always creates a CAS policy called \"provisioned\".\n\nPossible metrics values: -1, 0, 1. The value -1 indicates that the metric could not be calculated. The value 0 indicates the status field value \"No\". The value 1 indicates the status field value \"Yes\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "red", "index": 1, "text": "No" }, "1": { "color": "green", "index": 0, "text": "Yes" }, "-1": { "color": "#808080", "index": 2, "text": "Unable to calculate" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "red", "value": 0 }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 20 }, "id": 47, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_provisioned{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "CAS provisioned", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 25 }, "id": 45, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Phase", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"phase\". Status field description: The phase of the CAS.\n\nThe value Deleted indicates that the CAS has been deleted.\n\nThe value HEALTHY indicates that the CAS is running without any detected issues.\n\nThe value Provisioned indicates that the CAS is provisioned, its state is PENDING.\n\nThe value Unprovisioned indicates that the CAS is not provisioned.\n\nThe value EnablingMigration indicates that the CAS is provisioned but at least one node has not yet registered a backup CASes as requested.\n\nThe value Uninitialized indicates that the CAS is not provisioned and its state is PENDING.\n\nThe value AttestationFailure indicates that the CAS is provisioned but the attestation failed.\n\nThe value SnapshotFailure indicates the CAS is provisioned but its snapshots are not being performed in requested manner.\n\nThe value UNHEALTHY indicates that although the CAS is provisioned, it can be attested and its snapshots are being performed as requested, its state is UNHEALTHY. \n\nPossible metrics values: -1, a float64 between 0 and 1. The value -1 indicates that the metric could not be calculated. Otherwise, the string value of the status field \"phase\" is mapped to a float64 value between 0 and 1, depending on the string value's index in the ordered list UNHEALTHY, SnapshotFailure, AttestationFailure, EnablingMigration, Uninitialized, Unprovisioned, Provisioned, HEALTHY, Deleted.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "UNHEALTHY" }, "1": { "color": "#808080", "index": 8, "text": "Deleted" }, "0.125": { "color": "red", "index": 1, "text": "SnapshotFailure" }, "0.25": { "color": "red", "index": 2, "text": "AttestationFailure" }, "0.375": { "color": "yellow", "index": 3, "text": "EnablingMigration" }, "0.5": { "color": "yellow", "index": 4, "text": "Uninitialized" }, "0.625": { "color": "yellow", "index": 5, "text": "Unprovisioned" }, "0.75": { "color": "green", "index": 6, "text": "Provisioned" }, "0.875": { "color": "green", "index": 7, "text": "HEALTHY" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 27 }, "id": 43, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_phase{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CAS phase", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"phase\". Status field description: The phase of the CAS.\n\nThe value Deleted indicates that the CAS has been deleted.\n\nThe value HEALTHY indicates that the CAS is running without any detected issues.\n\nThe value Provisioned indicates that the CAS is provisioned, its state is PENDING.\n\nThe value Unprovisioned indicates that the CAS is not provisioned.\n\nThe value EnablingMigration indicates that the CAS is provisioned but at least one node has not yet registered a backup CASes as requested.\n\nThe value Uninitialized indicates that the CAS is not provisioned and its state is PENDING.\n\nThe value AttestationFailure indicates that the CAS is provisioned but the attestation failed.\n\nThe value SnapshotFailure indicates the CAS is provisioned but its snapshots are not being performed in requested manner.\n\nThe value UNHEALTHY indicates that although the CAS is provisioned, it can be attested and its snapshots are being performed as requested, its state is UNHEALTHY. \n\nPossible metrics values: -1, a float64 between 0 and 1. The value -1 indicates that the metric could not be calculated. Otherwise, the string value of the status field \"phase\" is mapped to a float64 value between 0 and 1, depending on the string value's index in the ordered list UNHEALTHY, SnapshotFailure, AttestationFailure, EnablingMigration, Uninitialized, Unprovisioned, Provisioned, HEALTHY, Deleted.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 3, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "UNHEALTHY" }, "1": { "color": "#808080", "index": 8, "text": "Deleted" }, "0.125": { "color": "red", "index": 1, "text": "SnapshotFailure" }, "0.25": { "color": "red", "index": 2, "text": "AttestationFailure" }, "0.375": { "color": "yellow", "index": 3, "text": "EnablingMigration" }, "0.5": { "color": "yellow", "index": 4, "text": "Uninitialized" }, "0.625": { "color": "yellow", "index": 5, "text": "Unprovisioned" }, "0.75": { "color": "green", "index": 6, "text": "Provisioned" }, "0.875": { "color": "green", "index": 7, "text": "HEALTHY" } }, "type": "value" } ], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 0 }, { "color": "#EAB839", "value": 0.375 }, { "color": "green", "value": 0.75 }, { "color": "#808080", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 18, "x": 6, "y": 27 }, "id": 121, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_phase{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "interval": "", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "CAS phase", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 33 }, "id": 48, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Pod disruption budget", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"podDisruptionBudgetAlert\".\n\nStatus field description: The number of its pods that the CAS is allowing to be shutdown, for example to allow for draining a node.\n\nThe value should always be equal to 0. In case it is 1, the pod might be terminated and rescheduled on a different node.\n\nIf no nodes exist that have been registered by the CAS' backup-controller, the pod will not come up, until a node where the CAS has been registered joins the cluster.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "pods" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 35 }, "id": 49, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_pod_disruption_budget_alert{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Number of its pods that the CAS is allowing to be shut down", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 43 }, "id": 124, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Replica alert", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"replicaAlert\".\n\nStatus field description: The number of expected CAS pods that are not running. (The number of expected CAS pods is always 1.) The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "pods" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 45 }, "id": 122, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_replica_alert{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Number of expected CAS pods that are not running", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 53 }, "id": 123, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Migration controller", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"numMigratableNodes\".\n\nStatus field description: The number of nodes with a successfully registered backup CAS.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "id": 126, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_num_migratable_nodes{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Number of nodes with a registered backup CAS", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"migrationNodeAlert\".\n\nStatus field description: The number of nodes without a successfully registered backup CAS. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "id": 127, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_migration_node_alert{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Number of nodes without a registered backup CAS", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"migrationControllerReplicaAlert\".\n\nStatus field description: The number of expected CAS backup-controller pods that are not running. (The number of expected CAS backup-controller pods is always 1.) The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "pods" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 63 }, "id": 128, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_migration_controller_replica_alert{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Number of expected CAS backup-controller pods that are not running", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 71 }, "id": 129, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Safety service", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"safetyServiceStatus\".\n\nStatus field description: The safety service status. It assumes the least healthy state between `safety_service_deployment_status` and `safety_service_pdb_status`\n\nPossible metrics values: -1, 0, 1, 2, 3. The value -1 indicates the status field value \"Unknown\". The value 0 indicates the status field value \"Disabled\". The value 1 indicates the status field value \"Unhealthy\". The value 2 indicates the status field value \"Alert\". The value 3 indicates the status field value \"Healthy\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "#808080", "index": 1, "text": "Disabled" }, "1": { "color": "red", "index": 2, "text": "Unheathy" }, "2": { "color": "yellow", "index": 3, "text": "Alert" }, "3": { "color": "green", "index": 4, "text": "Healthy" }, "-1": { "color": "#808080", "index": 0, "text": "Unknown" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "#808080", "value": 0 }, { "color": "red", "value": 1 }, { "color": "#EAB839", "value": 2 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 73 }, "id": 131, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_safety_service_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Safety Service Status", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"safetyServiceStatus\".\n\nStatus field description: The safety service status. It assumes the least healthy state between `safety_service_deployment_status` and `safety_service_pdb_status`\n\nPossible metrics values: -1, 0, 1, 2, 3. The value -1 indicates the status field value \"Unknown\". The value 0 indicates the status field value \"Disabled\". The value 1 indicates the status field value \"Unhealthy\". The value 2 indicates the status field value \"Alert\". The value 3 indicates the status field value \"Healthy\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "#808080", "index": 1, "text": "Disabled" }, "1": { "color": "red", "index": 2, "text": "Unheathy" }, "2": { "color": "yellow", "index": 3, "text": "Alert" }, "3": { "color": "green", "index": 4, "text": "Healthy" }, "-1": { "color": "#808080", "index": 0, "text": "Unknown" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "#808080", "value": 0 }, { "color": "red", "value": 1 }, { "color": "#EAB839", "value": 2 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 73 }, "id": 130, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_safety_service_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Safety Service Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"safetyServiceDeploymentStatus\".\n\nStatus field description: The status of the safety service deployment.\n\nThe value Healthy indicates that current number of ready replicas is equals to the desired number of replicas.\n\nThe value Alert indicates that safety service has pending replicas and ready replicas count is less than desired replicas count.\n\nThe value Unhealthy indicates that either current ready replicas count is less than spec.safetyService.minReplicas or deployment differs from spec. Also unhealthy if deployment if deployment has unavailable replicas when number of ready replicas count is not less than desired replicas count.\n\nThe value Disabled is set when safety service is disabled.\n\nThe value Unknown is set when there current status could not be determined.\n\nPossible metrics values: -1, 0, 1, 2, 3. The value -1 indicates the status field value \"Unknown\". The value 0 indicates the status field value \"Disabled\". The value 1 indicates the status field value \"Unhealthy\". The value 2 indicates the status field value \"Alert\". The value 3 indicates the status field value \"Healthy\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "#808080", "index": 1, "text": "Disabled" }, "1": { "color": "red", "index": 2, "text": "Unheathy" }, "2": { "color": "yellow", "index": 3, "text": "Alert" }, "3": { "color": "green", "index": 4, "text": "Healthy" }, "-1": { "color": "#808080", "index": 0, "text": "Unknown" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "#808080", "value": 0 }, { "color": "red", "value": 1 }, { "color": "#EAB839", "value": 2 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 78 }, "id": 134, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_safety_service_deployment_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Safety Service Deployment Status", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"safetyServiceDeploymentStatus\".\n\nStatus field description: The status of the safety service deployment.\n\nThe value Healthy indicates that current number of ready replicas is equals to the desired number of replicas.\n\nThe value Alert indicates that safety service has pending replicas and ready replicas count is less than desired replicas count.\n\nThe value Unhealthy indicates that either current ready replicas count is less than spec.safetyService.minReplicas or deployment differs from spec. Also unhealthy if deployment if deployment has unavailable replicas when number of ready replicas count is not less than desired replicas count.\n\nThe value Disabled is set when safety service is disabled.\n\nThe value Unknown is set when there current status could not be determined.\n\nPossible metrics values: -1, 0, 1, 2, 3. The value -1 indicates the status field value \"Unknown\". The value 0 indicates the status field value \"Disabled\". The value 1 indicates the status field value \"Unhealthy\". The value 2 indicates the status field value \"Alert\". The value 3 indicates the status field value \"Healthy\".", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "#808080", "index": 1, "text": "Disabled" }, "1": { "color": "red", "index": 2, "text": "Unheathy" }, "2": { "color": "yellow", "index": 3, "text": "Alert" }, "3": { "color": "green", "index": 4, "text": "Healthy" }, "-1": { "color": "#808080", "index": 0, "text": "Unknown" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "#808080", "value": 0 }, { "color": "red", "value": 1 }, { "color": "#EAB839", "value": 2 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 78 }, "id": 135, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_safety_service_deployment_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Safety Service Deployment Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"safetyServicePDBStatus\".\n\nStatus field description: The status of the safety service pod disruption budget.\n\nThe value Healthy indicates that pod disruption budget fo the safety service is ok.\n\nThe value Alert indicates that pod disruption has no more allowed disruptions.\n\nThe value Unhealthy indicates that either the current healthy pods count is less than desired healthy pods count or pdb differs from spec.\n\nThe value Disabled is set when safety service is disabled.\n\nThe value Unknown is set when there current status could not be determined.\n\nPossible metrics values: -1, 0, 1, 2, 3. The value -1 indicates the status field value \"Unknown\". The value 0 indicates the status field value \"Disabled\". The value 1 indicates the status field value \"Unhealthy\". The value 2 indicates the status field value \"Alert\". The value 3 indicates the status field value \"Healthy\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "#808080", "index": 1, "text": "Disabled" }, "1": { "color": "red", "index": 2, "text": "Unheathy" }, "2": { "color": "yellow", "index": 3, "text": "Alert" }, "3": { "color": "green", "index": 4, "text": "Healthy" }, "-1": { "color": "#808080", "index": 0, "text": "Unknown" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "#808080", "value": 0 }, { "color": "red", "value": 1 }, { "color": "#EAB839", "value": 2 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 83 }, "id": 132, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_safety_service_pdb_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Safety Service Pod Disruption Budget Status", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"safetyServicePDBStatus\".\n\nStatus field description: The status of the safety service pod disruption budget.\n\nThe value Healthy indicates that pod disruption budget fo the safety service is ok.\n\nThe value Alert indicates that pod disruption has no more allowed disruptions.\n\nThe value Unhealthy indicates that either the current healthy pods count is less than desired healthy pods count or pdb differs from spec.\n\nThe value Disabled is set when safety service is disabled.\n\nThe value Unknown is set when there current status could not be determined.\n\nPossible metrics values: -1, 0, 1, 2, 3. The value -1 indicates the status field value \"Unknown\". The value 0 indicates the status field value \"Disabled\". The value 1 indicates the status field value \"Unhealthy\". The value 2 indicates the status field value \"Alert\". The value 3 indicates the status field value \"Healthy\".", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "#808080", "index": 1, "text": "Disabled" }, "1": { "color": "red", "index": 2, "text": "Unheathy" }, "2": { "color": "yellow", "index": 3, "text": "Alert" }, "3": { "color": "green", "index": 4, "text": "Healthy" }, "-1": { "color": "#808080", "index": 0, "text": "Unknown" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "#808080", "value": 0 }, { "color": "red", "value": 1 }, { "color": "#EAB839", "value": 2 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 83 }, "id": 136, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_safety_service_pdb_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Safety Service Pod Disruption Budget Status", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 88 }, "id": 137, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Snapshot", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"snapshotStatus\".\n\nStatus field description: The snapshot status of the CAS.\n\nThe value Persisted indicates that snapshots of the CAS DB are successfully being performed and saved on persistent storage as requested (i.e., cas.spec.databaseSnapshots.persistence.enabled == true).\n\nThe value Ephemeral indicates that snapshots of the CAS DB are successfully being performed and saved on ephemeral storage as requested (i.e., cas.spec.databaseSnapshots.persistence.enabled == false).\n\nThe value Initializing indicates that no snapshot of the CAS DB has been made yet but the requested snapshot interval has not yet expired.\n\nThe value Disabled indicates that cas.spec.databaseSnapshots.enabled == false and that no snapshots are being taken of the CAS DB as requested.\n\nThe value Failed indicates that the snapshots of the CAS DB are not performed as requested.\n\nPossible metrics values: -1, a float64 between 0 and 1. The value -1 indicates that the metric could not be calculated. Otherwise, the string value of the status field \"snapshotStatus\" is mapped to a float64 value between 0 and 1, depending on the string value's index in the ordered list Failed, Disabled, Initializing, Ephemeral, Persisted.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "Failed" }, "1": { "color": "green", "index": 4, "text": "Persisted" }, "-1": { "color": "#808080", "index": 5, "text": "Unable to calculate" }, "0.25": { "color": "#F2495C", "index": 1, "text": "Disabled" }, "0.5": { "color": "yellow", "index": 2, "text": "Initializing" }, "0.75": { "color": "green", "index": 3, "text": "Ephemeral" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "red", "value": 0 }, { "color": "#EAB839", "value": 0.5 }, { "color": "green", "value": 0.75 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 90 }, "id": 140, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Snapshot Status", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric corresponds to the status field \"snapshotStatus\".\n\nStatus field description: The snapshot status of the CAS.\n\nThe value Persisted indicates that snapshots of the CAS DB are successfully being performed and saved on persistent storage as requested (i.e., cas.spec.databaseSnapshots.persistence.enabled == true).\n\nThe value Ephemeral indicates that snapshots of the CAS DB are successfully being performed and saved on ephemeral storage as requested (i.e., cas.spec.databaseSnapshots.persistence.enabled == false).\n\nThe value Initializing indicates that no snapshot of the CAS DB has been made yet but the requested snapshot interval has not yet expired.\n\nThe value Disabled indicates that cas.spec.databaseSnapshots.enabled == false and that no snapshots are being taken of the CAS DB as requested.\n\nThe value Failed indicates that the snapshots of the CAS DB are not performed as requested.\n\nPossible metrics values: -1, a float64 between 0 and 1. The value -1 indicates that the metric could not be calculated. Otherwise, the string value of the status field \"snapshotStatus\" is mapped to a float64 value between 0 and 1, depending on the string value's index in the ordered list Failed, Disabled, Initializing, Ephemeral, Persisted.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "Failed" }, "1": { "color": "green", "index": 4, "text": "Persisted" }, "-1": { "color": "#808080", "index": 5, "text": "Unable to calculate" }, "0.25": { "color": "#F2495C", "index": 1, "text": "Disabled" }, "0.5": { "color": "yellow", "index": 2, "text": "Initializing" }, "0.75": { "color": "green", "index": 3, "text": "Ephemeral" } }, "type": "value" } ], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#808080", "value": -1 }, { "color": "red", "value": 0 }, { "color": "#EAB839", "value": 0.5 }, { "color": "green", "value": 0.75 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 90 }, "id": 141, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_status{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Snapshot Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "A binary value indicating whether the snapshots of the CAS database are made as requested.\n\nPossible metrics values: 0, 1. The value 0 indicates success and 1 failure.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "green", "index": 0, "text": "Success" }, "1": { "color": "red", "index": 1, "text": "Failure" } }, "type": "value" } ], "max": 3, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 95 }, "id": 138, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_alert{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Snapshot Alert", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "A binary value indicating whether the snapshots of the CAS database are made as requested.\n\nPossible metrics values: 0, 1. The value 0 indicates success and 1 failure.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "green", "index": 0, "text": "Success" }, "1": { "color": "red", "index": 1, "text": "Failure" } }, "type": "value" } ], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 95 }, "id": 139, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_alert{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "{{__name__}} - {{cr_name}}", "range": true, "refId": "A" } ], "title": "Snapshot Alert", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Timestamp of the most recent snapshot of the CAS DB.\n\nPossible metrics values: -1 or timestamp in seconds. -1 indicates there is no snapshot yet.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 100 }, "id": 142, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^Time$/", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_timestamp{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Most recent CAS DB snapshot timestamp (since 1970)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The time when the snapshot was either last performed or omitted due to lack of change of the CAS DB.\n\nPossible metrics values: -1 or timestamp in seconds. -1 indicates there is no snapshot yet.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 100 }, "id": 143, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^Time$/", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_checked_timestamp{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"} ", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CAS DB snapshot checked timestamp (since 1970)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Current value of snapshot interval configured for CAS (seconds).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 108 }, "id": 144, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_cas_snapshot_configured_interval{cr_name=\"$cas_instance\",cr_namespace=\"$cas_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Snapshot interval configured", "type": "stat" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 116 }, "id": 23, "panels": [ { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 3, "w": 24, "x": 0, "y": 117 }, "id": 24, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "# LAS metrics\n\nThis section contains metrics pertaining to the LAS.", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 120 }, "id": 27, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## State", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "uid": "prometheus" }, "description": "This metric corresponds to the status field \"state\".\n\nStatus field description: The state of the LAS.\n\n\nThe value HEALTHY indicates the LAS is running without any detected issues.\n\nThe value PENDING indicates the LAS is not yet healthy but no problems have been detected, and it is expected to become healthy without human intervention.\n\nThe value UNHEALTHY indicates a problem has been detected.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"UNHEALTHY\". The value 0 indicates the status field value \"PENDING\". The value 1 indicates the status field value \"HEALTHY\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "yellow", "index": 1, "text": "PENDING" }, "1": { "color": "green", "index": 2, "text": "HEALTHY" }, "-1": { "color": "red", "index": 0, "text": "UNHEALTHY" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "scone_sgxplugin_state" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 122 }, "id": 25, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_las_state{cr_name=\"las\",cr_namespace=\"$scone_system_namespace\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "LAS state", "type": "stat" }, { "datasource": { "uid": "prometheus" }, "description": "This metric corresponds to the status field \"state\".\n\nStatus field description: The state of the LAS.\n\n\nThe value HEALTHY indicates the LAS is running without any detected issues.\n\nThe value PENDING indicates the LAS is not yet healthy but no problems have been detected, and it is expected to become healthy without human intervention.\n\nThe value UNHEALTHY indicates a problem has been detected.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"UNHEALTHY\". The value 0 indicates the status field value \"PENDING\". The value 1 indicates the status field value \"HEALTHY\".", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "yellow", "index": 1, "text": "PENDING" }, "1": { "color": "green", "index": 2, "text": "HEALTHY" }, "-1": { "color": "red", "index": 0, "text": "UNHEALTHY" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": -1 }, { "color": "#EAB839", "value": 0 }, { "color": "green", "value": 1 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "scone_sgxplugin_state" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "scone_las_state" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 122 }, "id": 26, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_las_state{cr_name=\"las\",cr_namespace=\"$scone_system_namespace\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "LAS state", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 127 }, "id": 31, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Number of nodes", "mode": "markdown" }, "pluginVersion": "12.2.1", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"desiredNumberScheduled\".\n\nStatus field description: The number of nodes on which a LAS pod has been selected to run. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 129 }, "id": 28, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#las-specific-metrics-and-alerts" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_las_desired_number_scheduled{cr_name=\"las\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of nodes on which a LAS pod has been selected to run", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"numberAvailable\".\n\nStatus field description: The number of nodes with a LAS pod in Ready state. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 129 }, "id": 29, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#las-specific-metrics-and-alerts" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_las_number_available{cr_name=\"las\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of nodes with a LAS pod in Ready state", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"lasReplicaAlert\".\n\nStatus field description: The number of expected LAS pods that are not running. The value -1 indicates that the metric could not be calculated or that no pods were expected to run, which is also considered erroneous.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": -1 }, { "color": "green", "value": 0 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 137 }, "id": 30, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_las_las_replica_alert{cr_name=\"las\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of expected LAS pods that are not running", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"lasProbeAlert\".\n\nStatus field description: The number of desired replicas, where the LAS probe is failing.\n\nThe value -1 indicates that the probe could not be run (e.g., due to a pod not being ready) on at least one of the nodes.\n\nThe value 0 indicates that the LAS probe terminated successfully on all nodes. A value > 0 conveys the number of nodes on which the probe command failed, i.e., returned with a non-zero exit code.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "replicas" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 137 }, "id": 32, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", "expr": "scone_las_las_probe_alert{cr_name=\"las\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of desired replicas where the LAS probe is failing", "type": "timeseries" } ], "title": "LAS metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 117 }, "id": 7, "panels": [ { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 3, "w": 24, "x": 0, "y": 116 }, "id": 1, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "# SGXPlugin metrics\n\nThis section contains metrics pertaining to the SGX Plugin.", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1285 }, "id": 14, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## State", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "uid": "prometheus" }, "description": "This metric corresponds to the status field \"state\".\n\nStatus field description: The state of the SGXPlugin.\n\nThe value HEALTHY indicates the SGXPlugin is running without any detected issues.\n\nThe value PENDING indicates the SGXPlugin is not yet healthy but no problems have been detected, and it is expected to become healthy without human intervention.\n\nThe value UNHEALTHY indicates a problem has been detected.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"UNHEALTHY\". The value 0 indicates the status field value \"PENDING\". The value 1 indicates the status field value \"HEALTHY\".\n", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "yellow", "index": 1, "text": "PENDING" }, "1": { "color": "green", "index": 2, "text": "HEALTHY" }, "-1": { "color": "red", "index": 0, "text": "UNHEALTHY" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "scone_sgxplugin_state" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1287 }, "id": 2, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts_1" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_sgxplugin_state{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "SGXPlugin state", "type": "stat" }, { "datasource": { "uid": "prometheus" }, "description": "This metric corresponds to the status field \"state\".\n\nStatus field description: The state of the SGXPlugin.\n\nThe value HEALTHY indicates the SGXPlugin is running without any detected issues.\n\nThe value PENDING indicates the SGXPlugin is not yet healthy but no problems have been detected, and it is expected to become healthy without human intervention.\n\nThe value UNHEALTHY indicates a problem has been detected.\n\nPossible metrics values: -1, 0, 1. The value -1 indicates the status field value \"UNHEALTHY\". The value 0 indicates the status field value \"PENDING\". The value 1 indicates the status field value \"HEALTHY\".\n", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "color": "yellow", "index": 1, "text": "PENDING" }, "1": { "color": "green", "index": 2, "text": "HEALTHY" }, "-1": { "color": "red", "index": 0, "text": "UNHEALTHY" } }, "type": "value" } ], "max": 1, "min": -1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": -1 }, { "color": "#EAB839", "value": 0 }, { "color": "green", "value": 1 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "scone_sgxplugin_state" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 18, "x": 6, "y": 1287 }, "id": 3, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts_1" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_sgxplugin_state{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "SGXPlugin state", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1292 }, "id": 15, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## SGX device usage", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"usedSGXDevices\".\n\nStatus field description: The total number of pods that has gotten access to an SGX device. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "devices" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1860 }, "id": 6, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_used_sgx_devices{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"} ", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Used SGX devices", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"usedSGXDevicesPerNode\".\n\nStatus field description: The number of pods that has gotten access to an SGX device on a node. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "devices" }, "overrides": [] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 1860 }, "id": 8, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_used_sgx_devices_per_node", "legendFormat": "{{nodeName}}", "range": true, "refId": "A" } ], "title": "Used SGX devices per node", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"availableSGXDevices\".\n\nStatus field description: The total number of SGX devices that are allocatable but not used. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "devices" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1868 }, "id": 5, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_available_sgx_devices{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Allocatable (but not used) SGX devices", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"availableSGXDevicesPerNode\".\n\nStatus field description: The number of SGX devices that are allocatable but not used at a node. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "devices" }, "overrides": [] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 1868 }, "id": 4, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_sgxplugin_available_sgx_devices_per_node", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{nodeName}}", "range": true, "refId": "A" } ], "title": "Allocatable (but not used) SGX devices per node", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"capacitySGXDevices\".\n\nStatus field description: The total number of SGX devices allocatable by pods. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "devices" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1876 }, "id": 9, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "scone_sgxplugin_capacity_sgx_devices{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "format": "time_series", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "SGX devices capacity", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"capacitySGXDevicesPerNode\".\n\nStatus field description: The total number of SGX devices that can be allocated by pods on a node. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "devices" }, "overrides": [] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 1876 }, "id": 10, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_capacity_sgx_devices_per_node", "legendFormat": "{{nodeName}}", "range": true, "refId": "A" } ], "title": "SGX devices capacity per Node", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1884 }, "id": 16, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## SGX-capable node metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"healthyServingReplicas\".\n\nStatus field description: The number of selected SGX-capable nodes with available SGX devices. The value -1 indicates that it could not be calculated. \n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 9, "x": 0, "y": 1886 }, "id": 18, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_healthy_serving_replicas{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of selected SGX-capable nodes with available SGX devices", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"sgxDeviceAlert\".\n\nStatus field description: The number of selected SGX-capable nodes, which has no available devices in the particular device category. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 1 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 9, "x": 9, "y": 1886 }, "id": 11, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts_1" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_sgx_device_alert", "legendFormat": "{{sgxDevice}}", "range": true, "refId": "A" } ], "title": "Number of selected SGX-capable nodes with no available devices in the particular device category", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"desiredReplicas\".\n\nStatus field description: The number of nodes on which the SGXPlugin has been selected to run. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "nodes" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "{__name__=\"scone_sgxplugin_desired_replicas\", container=\"manager\", cr_name=\"sgxplugin\", cr_namespace=\"$scone_system_namespace\", endpoint=\"http\", instance=\"10.42.1.22:8080\", job=\"scone-controller-manager-metrics-service\", namespace=\"$scone_system_namespace\", pod=\"scone-controller-manager-7c6bdf5569-8rjlc\", service=\"scone-controller-manager-metrics-service\"}" ], "prefix": "All except:", "readOnly": true } }, "properties": [] } ] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1886 }, "id": 13, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_desired_replicas{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Number of nodes on which the SGXPlugin has been selected to run", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"readyReplicas\".\n\nStatus field description: The number of nodes with a SGXPlugin pod in Ready state. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1894 }, "id": 19, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#metrics" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_ready_replicas{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of nodes with a SGXPlugin pod in Ready state", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"servingReplicas\".\n\nStatus field description: The number of SGX-capable nodes on which the SGXPlugin has been selected to run. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1894 }, "id": 20, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_serving_replicas{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of SGX-capable nodes on which the SGXPlugin has been selected to run", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"servingReplicaRatio\".\n\nStatus field description: The ratio of selected SGX-capable nodes with available SGX devices to the total of selected SGX-capable nodes (HealthyServingReplicas/ServingReplicas).\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.\n\nPossible metric values: -1, float64 between 0 and 1. The value -1 indicates that it could not be calculated.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1902 }, "id": 21, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_serving_replica_ratio{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Ratio of selected SGX-capable nodes with available SGX devices", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "This metric monitors to the status field \"sgxReplicaAlert\".\n\nStatus field description: The number of selected SGX-capable nodes with unavailable SGXPlugin pods. The value -1 indicates that it could not be calculated.\n\nPossible metrics values: The value of this metric is always equal to the value of the status field.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 1 } ] }, "unit": "nodes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1902 }, "id": 22, "links": [ { "targetBlank": true, "title": "SCONE documentation", "url": "https://sconedocs.github.io/3_resource_definitions/#alerts_1" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "scone_sgxplugin_sgx_replica_alert{cr_name=\"sgxplugin\",cr_namespace=\"$scone_system_namespace\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Number of selected SGX-capable nodes with unavailable SGXPlugin pods", "type": "timeseries" } ], "title": "SGXPlugin metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 118 }, "id": 50, "panels": [ { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 1554 }, "id": 51, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "# SCONE operator controller metrics\n\nThis section contains metrics pertaining to the SCONE operator controller, including:\n\n* Certificate read metrics\n* Controller runtime metrics\n* Go metrics\n* Leader election metrics\n* Process metrics\n* REST client request metrics\n* Workqueue metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1561 }, "id": 53, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Certificate read metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of certificate read errors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "read errors" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1563 }, "id": 52, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "certwatcher_read_certificate_errors_total", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Certificate read errors", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of certificate reads", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reads" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1563 }, "id": 54, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "certwatcher_read_certificate_total", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Total certificate reads", "type": "stat" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1571 }, "id": 55, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Controller runtime metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of currently used workers per controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "workers" }, "overrides": [ { "matcher": { "id": "byName", "options": "vault" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1573 }, "id": 56, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "controller_runtime_active_workers", "format": "time_series", "instant": false, "legendFormat": "{{controller}}", "range": true, "refId": "A" } ], "title": "Controller runtime active workers per controller", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1581 }, "id": 73, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "### Reconciliation metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of reconciliation errors per controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "errors" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1583 }, "id": 58, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_reconcile_errors_total", "legendFormat": "{{controller}}", "range": true, "refId": "A" } ], "title": "Reconciliation errors per controller", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of reconciliation panics per controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "panics" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1583 }, "id": 59, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_reconcile_panics_total", "legendFormat": "{{controller}}", "range": true, "refId": "A" } ], "title": "Reconciliation panics per controller", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of terminal reconciliation errors per controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "errors" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1591 }, "id": 60, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_terminal_reconcile_errors_total", "legendFormat": "{{controller}}", "range": true, "refId": "A" } ], "title": "Terminal reconciliation errors per controller", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of reconciliations per controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reconciliations" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1591 }, "id": 61, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_reconcile_total\n", "legendFormat": "{{controller}}: {{result}}", "range": true, "refId": "A" } ], "title": "Reconciliations per controller", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Maximum number of concurrent reconciles per controller", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reconciles" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1599 }, "id": 62, "options": { "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_max_concurrent_reconciles", "legendFormat": "{{controller}}", "range": true, "refId": "A" } ], "title": "Max concurrent reconciles", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Length of time per reconciliation.\n\n99% of reconciliation operations complete within this amount of seconds.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1607 }, "id": 63, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket[$__rate_interval])) by (le, controller))", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Reconcile duration - 99th percentile", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Length of time per reconciliation.\n\n50% of reconciliation operations complete within this amount of seconds.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1615 }, "id": 67, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum(rate(controller_runtime_reconcile_time_seconds_bucket[$__rate_interval])) by (controller, le))", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Reconcile duration - 50th percentile", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1623 }, "id": 74, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "### Webhoook metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of webhook panics", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "panics" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1625 }, "id": 64, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_webhook_panics_total", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "Webhook panics", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Current number of admission requests being served.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "requests" }, "overrides": [] }, "gridPos": { "h": 8, "w": 11, "x": 0, "y": 1633 }, "id": 65, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_webhook_requests_in_flight", "legendFormat": "{{webhook}}", "range": true, "refId": "A" } ], "title": "Webhook requests in flight", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of admission requests by HTTP status code.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "requests" }, "overrides": [] }, "gridPos": { "h": 8, "w": 13, "x": 11, "y": 1633 }, "id": 66, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "controller_runtime_webhook_requests_total", "legendFormat": "{{webhook}} - status code {{code}}", "range": true, "refId": "A" } ], "title": "Admission requests by HTTP status code.", "type": "timeseries" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1641 }, "id": 68, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "## Go metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Information about the Go environment.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false, "minWidth": 150 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "node" }, "properties": [ { "id": "custom.width", "value": 200 } ] }, { "matcher": { "id": "byName", "options": "service" }, "properties": [ { "id": "custom.width", "value": 300 } ] }, { "matcher": { "id": "byName", "options": "pod" }, "properties": [ { "id": "custom.width", "value": 300 } ] }, { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.width", "value": 200 } ] }, { "matcher": { "id": "byName", "options": "container" }, "properties": [ { "id": "custom.width", "value": 180 } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "custom.width", "value": 300 } ] }, { "matcher": { "id": "byName", "options": "version" }, "properties": [ { "id": "custom.width", "value": 100 } ] }, { "matcher": { "id": "byName", "options": "metrics_path" }, "properties": [ { "id": "custom.width", "value": 75 } ] }, { "matcher": { "id": "byName", "options": "endpoint" }, "properties": [ { "id": "custom.width", "value": 125 } ] } ] }, "gridPos": { "h": 12, "w": 24, "x": 0, "y": 1643 }, "id": 69, "options": { "cellHeight": "sm", "footer": { "countRows": false, "enablePagination": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "node" } ] }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "go_info", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Go environment information", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": false, "Value": true, "__name__": true }, "includeByName": {}, "indexByName": {}, "renameByName": { "__name__": "" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of goroutines that currently exist.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "goroutines" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1655 }, "id": 70, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "sum(go_goroutines)", "format": "time_series", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total goroutines", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of OS threads created.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "threads" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1655 }, "id": 71, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "sum(go_threads)", "format": "time_series", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Go threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "A summary of the pause duration of garbage collection cycles.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 16, "w": 24, "x": 0, "y": 1663 }, "id": 72, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "go_gc_duration_seconds", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Garbage Collection duration", "transformations": [ { "id": "organize", "options": { "excludeByName": { "__name__": true }, "includeByName": {}, "indexByName": { "Time": 0, "Value": 12, "__name__": 1, "container": 2, "endpoint": 3, "instance": 4, "job": 5, "metrics_path": 10, "namespace": 6, "node": 11, "pod": 7, "quantile": 8, "service": 9 }, "renameByName": { "Value": "go_gc_duration_seconds", "metrics_path": "" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes allocated and still in use.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1679 }, "id": 75, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_alloc_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Allocated bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of bytes allocated, even if freed.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1679 }, "id": 77, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_alloc_bytes_total)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Allocated bytes (even if freed)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes used by the profiling bucket hash table.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1679 }, "id": 83, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_buck_hash_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Profiling bucket hash table bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes used for garbage collection system metadata.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1679 }, "id": 79, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_gc_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Garbage collection metadata bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of seconds since 1970 of last garbage collection.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "kubelet on 172.18.0.6:10250 k3d-cas-chaos-agent-1" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1687 }, "id": 80, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "go_memstats_last_gc_time_seconds", "legendFormat": "{{job}} on {{instance}} {{pod}} {{node}}", "range": true, "refId": "A" } ], "title": "Time since last garbage collection (since 1970)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of heap bytes when next garbage collection will take place.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "apiserver on 172.18.0.3:6443 " ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1687 }, "id": 81, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "go_memstats_next_gc_bytes", "legendFormat": "{{job}} on {{instance}} {{node}} {{pod}}", "range": true, "refId": "A" } ], "title": "Next garbage collection bytes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of frees.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "billion frees" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1695 }, "id": 78, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_frees_total) / 1e9", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total frees", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of pointer lookups.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "lookups" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1695 }, "id": 82, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_lookups_total)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Lookups", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of mallocs", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "billion mallocs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1695 }, "id": 84, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_mallocs_total) / 1e9", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Mallocs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of heap bytes released to OS.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1695 }, "id": 89, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_heap_released_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Released heap bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of heap bytes that are in use.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1703 }, "id": 87, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_heap_inuse_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Heap bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of heap bytes allocated and still in use.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1703 }, "id": 86, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_heap_alloc_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Allocated heap bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of heap bytes waiting to be used.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1703 }, "id": 88, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_heap_idle_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Idle heap bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of heap bytes obtained from system.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1703 }, "id": 91, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_heap_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Heap system bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of allocated heap objects.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "million objects" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1711 }, "id": 90, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_heap_objects) / 1e6", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Allocated heap objects", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes in use by the stack allocator.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1719 }, "id": 92, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_stack_inuse_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Stack bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes obtained from system for stack allocator.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1719 }, "id": 93, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_stack_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Stack system bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes in use by mcache structures.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1727 }, "id": 94, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_mcache_inuse_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "mcache structure bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes used for mcache structures obtained from system.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1727 }, "id": 95, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_mcache_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "mcache structure system bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes in use by mspan structures.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1727 }, "id": 96, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_mspan_inuse_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "mspan structure bytes in use", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes used for mspan structures obtained from system.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1727 }, "id": 97, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_mspan_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "mspan structure system bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes obtained from system.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1735 }, "id": 98, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "System bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of bytes used for other system allocations.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1735 }, "id": 99, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(go_memstats_other_sys_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Other system allocation bytes", "type": "stat" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1743 }, "id": 100, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "### Process metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "A" }, "properties": [ { "id": "mappings", "value": [ { "options": { "0": { "index": 0, "text": "Backup" }, "1": { "index": 1, "text": "Master" } }, "type": "value" } ] } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1745 }, "id": 101, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "leader_election_master_status", "legendFormat": "{{name}}", "range": true, "refId": "A" } ], "title": "Reporting system leader election status", "transformations": [ { "id": "groupBy", "options": { "fields": { "61d57791.scone.cloud": { "aggregations": [] }, "Time": { "aggregations": [ "lastNotNull" ] }, "leader_election_master_status": { "aggregations": [ "lastNotNull" ] } } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Start time of the process since UNIX epoch in seconds.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1753 }, "id": 105, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "process_start_time_seconds", "legendFormat": "{{job}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Start time (since 1970)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total user and system CPU time spent in seconds.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1753 }, "id": 102, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "process_cpu_seconds_total", "legendFormat": "{{job}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "CPU time spent", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of open file descriptors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "file descriptors" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1761 }, "id": 104, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "process_open_fds", "legendFormat": "{{job}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Open file descriptors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Maximum number of open file descriptors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "file descriptors" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1761 }, "id": 103, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "process_max_fds", "legendFormat": "{{job}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Maximum file descriptors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Virtual memory size in bytes.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1769 }, "id": 106, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(process_virtual_memory_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Virtual memory size", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Maximum amount of virtual memory available in bytes.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1769 }, "id": 107, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(process_virtual_memory_max_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Maximum virtual memory size", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Resident memory size in bytes.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1769 }, "id": 108, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "expr": "sum(process_resident_memory_bytes)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Resident memory size", "type": "stat" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1777 }, "id": 109, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "### REST client request metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Number of HTTP requests, partitioned by status code, method, and host.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.width", "value": 200 } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.width", "value": 210 } ] } ] }, "gridPos": { "h": 16, "w": 24, "x": 0, "y": 1779 }, "id": 110, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "rest_client_requests_total", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Number of HTTP requests", "transformations": [ { "id": "organize", "options": { "excludeByName": { "__name__": true }, "includeByName": {}, "indexByName": {}, "renameByName": {} } }, { "id": "groupBy", "options": { "fields": { "Time": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "Value": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "code": { "aggregations": [], "operation": "groupby" }, "container": { "aggregations": [], "operation": "aggregate" }, "endpoint": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "host": { "aggregations": [], "operation": "groupby" }, "instance": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "job": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "method": { "aggregations": [], "operation": "groupby" }, "metrics_path": { "aggregations": [], "operation": "aggregate" }, "namespace": { "aggregations": [], "operation": "aggregate" }, "node": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "pod": { "aggregations": [], "operation": "aggregate" }, "service": { "aggregations": [], "operation": "aggregate" } } } } ], "type": "table" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1795 }, "id": 115, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "### Workqueue metrics", "mode": "markdown" }, "pluginVersion": "12.0.2", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of adds handled by workqueue", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "adds" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1797 }, "id": 111, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "workqueue_adds_total", "format": "time_series", "instant": false, "legendFormat": "{{name}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Total adds", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total number of retries handled by workqueue", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "retries" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1797 }, "id": 119, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "workqueue_retries_total", "format": "time_series", "instant": false, "legendFormat": "{{name}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Total retries", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Current depth of workqueue", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1805 }, "id": 112, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "workqueue_depth", "format": "time_series", "instant": false, "legendFormat": "{{name}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Workqueue depth", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How many seconds of work has been done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads.\n\nOne can deduce the number of stuck threads by observing the rate at which this increases.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1805 }, "id": 120, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "workqueue_unfinished_work_seconds", "format": "time_series", "instant": false, "legendFormat": "{{name}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Unfinished work duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How many seconds has the longest running processor for workqueue been running.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1813 }, "id": 113, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "workqueue_longest_running_processor_seconds", "format": "time_series", "instant": false, "legendFormat": "{{name}} - {{endpoint}} on {{instance}}{{metrics_path}} {{node}}", "range": true, "refId": "A" } ], "title": "Longest running processor time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How long in seconds an item stays in workqueue before being requested.\n\n99% of items stay up to this amount of time in the workqueue.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1821 }, "id": 114, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket[$__rate_interval])) by (le, name))", "hide": false, "instant": false, "legendFormat": "{{label_name}}", "range": true, "refId": "A" } ], "title": "Queue duration - 99th percentile", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How long in seconds an item stays in workqueue before being requested.\n\n50% of items stay up to this amount of time in the workqueue.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1821 }, "id": 116, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum(rate(workqueue_queue_duration_seconds_bucket[$__rate_interval])) by (le, name))", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Queue duration - 50th percentile", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How long in seconds processing an item from workqueue takes.\n\n99% of items stay up to this amount of time in process.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1829 }, "id": 117, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket[$__rate_interval])) by (le, name))", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Processing duration - 99th percentile", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How long in seconds processing an item from workqueue takes.\n\n50% of items stay up to this amount of time in process.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1829 }, "id": 118, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum(rate(workqueue_work_duration_seconds_bucket[$__rate_interval])) by (le, name))", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Processing duration - 50th percentile", "type": "timeseries" } ], "title": "SCONE operator controller metrics", "type": "row" } ], "preload": false, "schemaVersion": 42, "tags": [], "templating": { "list": [ { "current": { "text": "osc-scone-system", "value": "osc-scone-system" }, "description": "Namespace where scone services reside", "label": "Scone System Namespace", "name": "scone_system_namespace", "options": [ { "selected": true, "text": "osc-scone-system", "value": "osc-scone-system" } ], "query": "osc-scone-system", "type": "custom" }, { "current": { "text": "osc-scone-system", "value": "osc-scone-system" }, "description": "Namespace where CAS resides", "label": "CAS Namespace", "name": "cas_namespace", "options": [ { "selected": true, "text": "osc-scone-system", "value": "osc-scone-system" } ], "query": "osc-scone-system", "type": "custom" }, { "current": { "text": "cas", "value": "cas" }, "definition": "label_values(scone_cas_state,cr_name)", "description": "The selected CAS instance", "includeAll": false, "label": "CAS Instance", "name": "cas_instance", "options": [], "query": { "qryType": 1, "query": "label_values(scone_cas_state,cr_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "sort": 7, "type": "query" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "SCONE Operator Dashboard", "uid": "23511b14-7903-4122-b737-59a439ea4914", "version": 8 }