From 50e49a926416fa11bafa8e676408686c6a03fbcf Mon Sep 17 00:00:00 2001 From: Snarf Date: Sun, 15 Nov 2026 12:43:09 -0500 Subject: [PATCH] Split cluster resource monitors into Aerodrome, Spaceport, Shipyard folders and dashboards --- .../grafana/aerodrome-resource-monitor.json | 1060 ++++++ .../grafana/shipyard-resource-monitor.json | 984 ++++++ .../grafana/spaceport-resource-monitor.json | 1060 ++++++ manifests/aerodrome/grafana/grafana.yaml | 2849 +++++++---------- 4 files changed, 4318 insertions(+), 1635 deletions(-) create mode 100644 dashboards/grafana/aerodrome-resource-monitor.json create mode 100644 dashboards/grafana/shipyard-resource-monitor.json create mode 100644 dashboards/grafana/spaceport-resource-monitor.json diff --git a/dashboards/grafana/aerodrome-resource-monitor.json b/dashboards/grafana/aerodrome-resource-monitor.json new file mode 100644 index 0000000..e42f05c --- /dev/null +++ b/dashboards/grafana/aerodrome-resource-monitor.json @@ -0,0 +1,1060 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "refresh": "15s", + "schemaVersion": 39, + "tags": [ + "aerodrome", + "resource-monitor", + "infrastructure" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "\u2708\ufe0f Aerodrome Resource Monitor", + "uid": "aerodrome-resource-monitor", + "version": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udda5\ufe0f Systems & Host Performance \u2014 Aerodrome Cluster Fleet", + "type": "row" + }, + { + "id": 1, + "title": "Online Nodes (Aerodrome)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(node_load1{cluster=\"aerodrome\"})", + "legendFormat": "Nodes", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Avg Cluster CPU Load (1m)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "avg(node_load1{cluster=\"aerodrome\"})", + "legendFormat": "Load", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 2.0 + }, + { + "color": "red", + "value": 4.0 + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Monitored RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(node_memory_MemTotal_bytes{cluster=\"aerodrome\"})", + "legendFormat": "RAM", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Cluster RAM In-Use %", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (sum(node_memory_MemAvailable_bytes{cluster=\"aerodrome\"}) / sum(node_memory_MemTotal_bytes{cluster=\"aerodrome\"}))) * 100", + "legendFormat": "In-Use", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 5, + "title": "CPU Utilization by Host (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"aerodrome\"}[2m])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 6, + "title": "1-Minute Load Average by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_load1{cluster=\"aerodrome\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "none", + "min": 0 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 7, + "title": "Memory Used % by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes{cluster=\"aerodrome\"} / node_memory_MemTotal_bytes{cluster=\"aerodrome\"})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 8, + "title": "Available RAM by Host (GiB)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes{cluster=\"aerodrome\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "min" + ] + } + } + }, + { + "id": 9, + "title": "Root (/) Filesystem Usage %", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\", cluster=\"aerodrome\"} / node_filesystem_size_bytes{mountpoint=\"/\", cluster=\"aerodrome\"})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 10, + "title": "Disk Read Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{cluster=\"aerodrome\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 11, + "title": "Disk Write Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{cluster=\"aerodrome\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 12, + "title": "Host Network Receive (RX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|flannel.*|cali.*\", cluster=\"aerodrome\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 13, + "title": "Host Network Transmit (TX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|flannel.*|cali.*\", cluster=\"aerodrome\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 200, + "title": "\ud83d\udce6 Kubernetes Containers & Workloads \u2014 Aerodrome Cluster", + "type": "row" + }, + { + "id": 14, + "title": "Monitored Containers", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(container_last_seen{container!=\"\", instance=~\"sninfr2.*\"})", + "legendFormat": "Containers", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 15, + "title": "Active Kubernetes Pods", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(count by (pod, namespace) (container_last_seen{container!=\"\", instance=~\"sninfr2.*\"}))", + "legendFormat": "Pods", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 16, + "title": "Total Container CPU Load", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\", instance=~\"sninfr2.*\"}[2m]))", + "legendFormat": "Cores", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 17, + "title": "Total Container Working RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"\", instance=~\"sninfr2.*\"})", + "legendFormat": "RAM", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 18, + "title": "Top 10 CPU Consuming Pods (Cores)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=\"\", instance=~\"sninfr2.*\"}[2m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 19, + "title": "CPU Throttling by Namespace", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{instance=~\"sninfr2.*\"}[2m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "none" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 20, + "title": "Top 10 Memory Consuming Pods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 50 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=\"\", instance=~\"sninfr2.*\"}))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 21, + "title": "Memory Working Set by Namespace", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 50 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=\"\", instance=~\"sninfr2.*\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "bytes" + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 22, + "title": "Top Pod Network RX (Receive)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{instance=~\"sninfr2.*\"}[2m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 23, + "title": "Top Pod Network TX (Transmit)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{instance=~\"sninfr2.*\"}[2m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + } + ] +} \ No newline at end of file diff --git a/dashboards/grafana/shipyard-resource-monitor.json b/dashboards/grafana/shipyard-resource-monitor.json new file mode 100644 index 0000000..9056594 --- /dev/null +++ b/dashboards/grafana/shipyard-resource-monitor.json @@ -0,0 +1,984 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "refresh": "15s", + "schemaVersion": 39, + "tags": [ + "shipyard", + "resource-monitor", + "infrastructure" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "\u2693 Shipyard Resource Monitor", + "uid": "shipyard-resource-monitor", + "version": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udda5\ufe0f Systems & Host Performance \u2014 Shipyard Cluster Fleet", + "type": "row" + }, + { + "id": 1, + "title": "Online Nodes (Shipyard)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(node_load1{cluster=\"shipyard\"})", + "legendFormat": "Nodes", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Avg Cluster CPU Load (1m)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "avg(node_load1{cluster=\"shipyard\"})", + "legendFormat": "Load", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 2.0 + }, + { + "color": "red", + "value": 4.0 + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Monitored RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(node_memory_MemTotal_bytes{cluster=\"shipyard\"})", + "legendFormat": "RAM", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Cluster RAM In-Use %", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (sum(node_memory_MemAvailable_bytes{cluster=\"shipyard\"}) / sum(node_memory_MemTotal_bytes{cluster=\"shipyard\"}))) * 100", + "legendFormat": "In-Use", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 5, + "title": "CPU Utilization by Host (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"shipyard\"}[2m])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 6, + "title": "1-Minute Load Average by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_load1{cluster=\"shipyard\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "none", + "min": 0 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 7, + "title": "Memory Used % by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes{cluster=\"shipyard\"} / node_memory_MemTotal_bytes{cluster=\"shipyard\"})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 8, + "title": "Available RAM by Host (GiB)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes{cluster=\"shipyard\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "min" + ] + } + } + }, + { + "id": 9, + "title": "Root (/) Filesystem Usage %", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\", cluster=\"shipyard\"} / node_filesystem_size_bytes{mountpoint=\"/\", cluster=\"shipyard\"})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 10, + "title": "Disk Read Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{cluster=\"shipyard\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 11, + "title": "Disk Write Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{cluster=\"shipyard\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 12, + "title": "Host Network Receive (RX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|flannel.*|cali.*\", cluster=\"shipyard\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 13, + "title": "Host Network Transmit (TX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|flannel.*|cali.*\", cluster=\"shipyard\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 200, + "title": "\ud83d\udce6 Docker Swarm Containers & Service Stacks \u2014 Shipyard Fleet", + "type": "row" + }, + { + "id": 14, + "title": "Monitored Swarm Containers", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(container_last_seen{cluster=\"shipyard\", name!=\"\"}) or count(container_last_seen{cluster=\"shipyard\"})", + "legendFormat": "Containers", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 15, + "title": "Active Swarm Nodes", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(up{cluster=\"shipyard\", job=\"node-exporter\"})", + "legendFormat": "Swarm Nodes", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 16, + "title": "Total Container CPU Load", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"shipyard\"}[2m]))", + "legendFormat": "Cores", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 17, + "title": "Total Container Working RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"shipyard\"})", + "legendFormat": "RAM", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 18, + "title": "Container CPU Usage by Service/Task", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (instance, id) (rate(container_cpu_usage_seconds_total{cluster=\"shipyard\"}[2m])))", + "legendFormat": "{{instance}} ({{id}})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 19, + "title": "Container Memory Working Set by Service/Task", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (instance, id) (container_memory_working_set_bytes{cluster=\"shipyard\"}))", + "legendFormat": "{{instance}} ({{id}})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 20, + "title": "Container Network Receive (RX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster=\"shipyard\"}[2m])))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 21, + "title": "Container Network Transmit (TX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster=\"shipyard\"}[2m])))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + } + ] +} \ No newline at end of file diff --git a/dashboards/grafana/spaceport-resource-monitor.json b/dashboards/grafana/spaceport-resource-monitor.json new file mode 100644 index 0000000..db13306 --- /dev/null +++ b/dashboards/grafana/spaceport-resource-monitor.json @@ -0,0 +1,1060 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "refresh": "15s", + "schemaVersion": 39, + "tags": [ + "spaceport", + "resource-monitor", + "infrastructure" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "\ud83d\ude80 Spaceport Resource Monitor", + "uid": "spaceport-resource-monitor", + "version": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udda5\ufe0f Systems & Host Performance \u2014 Spaceport Cluster Fleet", + "type": "row" + }, + { + "id": 1, + "title": "Online Nodes (Spaceport)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(node_load1{cluster=\"spaceport\"})", + "legendFormat": "Nodes", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Avg Cluster CPU Load (1m)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "avg(node_load1{cluster=\"spaceport\"})", + "legendFormat": "Load", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 2.0 + }, + { + "color": "red", + "value": 4.0 + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Monitored RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(node_memory_MemTotal_bytes{cluster=\"spaceport\"})", + "legendFormat": "RAM", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Cluster RAM In-Use %", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (sum(node_memory_MemAvailable_bytes{cluster=\"spaceport\"}) / sum(node_memory_MemTotal_bytes{cluster=\"spaceport\"}))) * 100", + "legendFormat": "In-Use", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 5, + "title": "CPU Utilization by Host (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"spaceport\"}[2m])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 6, + "title": "1-Minute Load Average by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_load1{cluster=\"spaceport\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "none", + "min": 0 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 7, + "title": "Memory Used % by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes{cluster=\"spaceport\"} / node_memory_MemTotal_bytes{cluster=\"spaceport\"})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 8, + "title": "Available RAM by Host (GiB)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes{cluster=\"spaceport\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "min" + ] + } + } + }, + { + "id": 9, + "title": "Root (/) Filesystem Usage %", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\", cluster=\"spaceport\"} / node_filesystem_size_bytes{mountpoint=\"/\", cluster=\"spaceport\"})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 10, + "title": "Disk Read Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{cluster=\"spaceport\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 11, + "title": "Disk Write Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 21 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{cluster=\"spaceport\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 12, + "title": "Host Network Receive (RX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|flannel.*|cali.*\", cluster=\"spaceport\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 13, + "title": "Host Network Transmit (TX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|flannel.*|cali.*\", cluster=\"spaceport\"}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 200, + "title": "\ud83d\udce6 Kubernetes Containers & Workloads \u2014 Spaceport Cluster", + "type": "row" + }, + { + "id": 14, + "title": "Monitored Containers", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(container_last_seen{container!=\"\", cluster=\"spaceport\"})", + "legendFormat": "Containers", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 15, + "title": "Active Kubernetes Pods", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(count by (pod, namespace) (container_last_seen{container!=\"\", cluster=\"spaceport\"}))", + "legendFormat": "Pods", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 16, + "title": "Total Container CPU Load", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\", cluster=\"spaceport\"}[2m]))", + "legendFormat": "Cores", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 17, + "title": "Total Container Working RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 38 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"\", cluster=\"spaceport\"})", + "legendFormat": "RAM", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 18, + "title": "Top 10 CPU Consuming Pods (Cores)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=\"\", cluster=\"spaceport\"}[2m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 19, + "title": "CPU Throttling by Namespace", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{cluster=\"spaceport\"}[2m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "none" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 20, + "title": "Top 10 Memory Consuming Pods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 50 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=\"\", cluster=\"spaceport\"}))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "lastNotNull", + "max" + ] + } + } + }, + { + "id": 21, + "title": "Memory Working Set by Namespace", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 50 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=\"\", cluster=\"spaceport\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "bytes" + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 22, + "title": "Top Pod Network RX (Receive)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{cluster=\"spaceport\"}[2m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + }, + { + "id": 23, + "title": "Top Pod Network TX (Transmit)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{cluster=\"spaceport\"}[2m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + } + } + ] +} \ No newline at end of file diff --git a/manifests/aerodrome/grafana/grafana.yaml b/manifests/aerodrome/grafana/grafana.yaml index d3af6e2..272b30d 100644 --- a/manifests/aerodrome/grafana/grafana.yaml +++ b/manifests/aerodrome/grafana/grafana.yaml @@ -117,15 +117,18 @@ spec: - mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml name: dashboards-provider-config subPath: dashboards.yaml - - mountPath: /var/lib/grafana-dashboards/ai-stack-dashboard.json + - mountPath: /var/lib/grafana-dashboards/ai-stack/ai-stack-dashboard.json name: ai-stack-dashboard subPath: ai-stack-dashboard.json - - mountPath: /var/lib/grafana-dashboards/systems-dashboard.json - name: systems-dashboard - subPath: systems-dashboard.json - - mountPath: /var/lib/grafana-dashboards/container-metrics-dashboard.json - name: container-metrics-dashboard - subPath: container-metrics-dashboard.json + - mountPath: /var/lib/grafana-dashboards/aerodrome/aerodrome-resource-monitor.json + name: aerodrome-dashboard + subPath: aerodrome-resource-monitor.json + - mountPath: /var/lib/grafana-dashboards/spaceport/spaceport-resource-monitor.json + name: spaceport-dashboard + subPath: spaceport-resource-monitor.json + - mountPath: /var/lib/grafana-dashboards/shipyard/shipyard-resource-monitor.json + name: shipyard-dashboard + subPath: shipyard-resource-monitor.json dnsPolicy: ClusterFirst enableServiceLinks: true initContainers: @@ -177,16 +180,22 @@ spec: defaultMode: 420 name: grafana-dashboards-provider name: dashboards-provider-config - - configMap: + - name: ai-stack-dashboard + configMap: defaultMode: 420 name: grafana-dashboard-ai-stack - name: ai-stack-dashboard - - configMap: - name: grafana-dashboard-systems - name: systems-dashboard - - configMap: - name: grafana-dashboard-containers - name: container-metrics-dashboard + - name: aerodrome-dashboard + configMap: + defaultMode: 420 + name: grafana-dashboard-aerodrome + - name: spaceport-dashboard + configMap: + defaultMode: 420 + name: grafana-dashboard-spaceport + - name: shipyard-dashboard + configMap: + defaultMode: 420 + name: grafana-dashboard-shipyard --- apiVersion: v1 kind: Service @@ -292,16 +301,9 @@ metadata: labels: app.kubernetes.io/name: grafana data: - datasources.yaml: | - apiVersion: 1 - datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - jsonData: - timeInterval: 15s + datasources.yaml: "apiVersion: 1\ndatasources:\n - name: Prometheus\n type:\ + \ prometheus\n access: proxy\n url: http://prometheus:9090\n isDefault:\ + \ true\n jsonData:\n timeInterval: 15s\n" --- apiVersion: v1 kind: ConfigMap @@ -311,17 +313,17 @@ metadata: labels: app.kubernetes.io/name: grafana data: - dashboards.yaml: | - apiVersion: 1 - providers: - - name: 'ai-stack-provider' - orgId: 1 - folder: 'AI Stack & Inference' - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana-dashboards + dashboards.yaml: "apiVersion: 1\nproviders:\n - name: 'ai-stack-provider'\n \ + \ orgId: 1\n folder: 'AI Stack & Inference'\n type: file\n disableDeletion:\ + \ false\n editable: true\n options:\n path: /var/lib/grafana-dashboards/ai-stack\n\ + \ - name: 'aerodrome-provider'\n orgId: 1\n folder: 'Aerodrome'\n type:\ + \ file\n disableDeletion: false\n editable: true\n options:\n path:\ + \ /var/lib/grafana-dashboards/aerodrome\n - name: 'spaceport-provider'\n orgId:\ + \ 1\n folder: 'Spaceport'\n type: file\n disableDeletion: false\n \ + \ editable: true\n options:\n path: /var/lib/grafana-dashboards/spaceport\n\ + \ - name: 'shipyard-provider'\n orgId: 1\n folder: 'Shipyard'\n type:\ + \ file\n disableDeletion: false\n editable: true\n options:\n path:\ + \ /var/lib/grafana-dashboards/shipyard\n" --- apiVersion: v1 kind: ConfigMap @@ -331,1620 +333,1197 @@ metadata: labels: app.kubernetes.io/name: grafana data: - ai-stack-dashboard.json: | - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "title": "\u26a1 AI Stack Executive Overview & Cloud Cost Offsets", - "type": "row" - }, - { - "id": 1, - "title": "Prompt Tokens Processed (In)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(vllm:prompt_tokens_total)", - "legendFormat": "Prompt Tokens", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "locale" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 2, - "title": "Generation Tokens Synthesized (Out)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(vllm:generation_tokens_total)", - "legendFormat": "Gen Tokens", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "locale" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 3, - "title": "Homelab Cloud API Savings ($USD)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", - "legendFormat": "Net Savings", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "currencyUSD" - } - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - } - }, - { - "id": 4, - "title": "Active GPU Cluster Health", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "count(DCGM_FI_DEV_GPU_TEMP)", - "legendFormat": "Tesla T4s Active", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "short" - } - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center" - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 200, - "title": "\ud83c\udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)", - "type": "row" - }, - { - "id": 5, - "title": "GPU Core Utilization (%)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL", - "legendFormat": "{{Hostname}} ({{modelName}})", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "percent", - "min": 0, - "max": 100 - } - } - }, - { - "id": 6, - "title": "VRAM Memory Allocated & Used", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", - "legendFormat": "{{Hostname}} VRAM Used", - "refId": "A" - }, - { - "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", - "legendFormat": "{{Hostname}} VRAM Total", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15 - }, - "unit": "bytes" - } - } - }, - { - "id": 7, - "title": "GPU Thermal Profile (\u00b0C)", - "type": "timeseries", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP", - "legendFormat": "{{Hostname}} (\u00b0C)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10 - }, - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 65 - }, - { - "color": "red", - "value": 80 - } - ] - } - } - } - }, - { - "id": 8, - "title": "GPU Power Draw (Watts)", - "type": "timeseries", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE", - "legendFormat": "{{Hostname}} (Watts)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15 - }, - "unit": "watt" - } - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 300, - "title": "\ud83e\udde0 vLLM Real-Time Inference Dynamics & Token Velocity", - "type": "row" - }, - { - "id": 9, - "title": "Token Generation Velocity (Tokens / Sec)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", - "legendFormat": "{{model_name}} (Out)", - "refId": "A" - }, - { - "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", - "legendFormat": "{{model_name}} (In)", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 25 - }, - "unit": "cps" - } - } - }, - { - "id": 10, - "title": "Inference Concurrency & Request Backlog", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "vllm:num_requests_running", - "legendFormat": "{{model_name}} Running", - "refId": "A" - }, - { - "expr": "vllm:num_requests_waiting", - "legendFormat": "{{model_name}} Queued", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 15 - }, - "unit": "short" - } - } - }, - { - "id": 11, - "title": "GPU KV Cache Memory Saturation", - "type": "timeseries", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "vllm:gpu_cache_usage_factor * 100", - "legendFormat": "{{model_name}} Cache Usage", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 30 - }, - "unit": "percent", - "min": 0, - "max": 100 - } - } - }, - { - "id": 12, - "title": "NVIDIA Triton & Qdrant Operations", - "type": "timeseries", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 30 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "rate(nv_inference_request_success[2m])", - "legendFormat": "Triton Success/s", - "refId": "A" - }, - { - "expr": "cpu_cores_used", - "legendFormat": "Qdrant CPU Cores", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth" - }, - "unit": "short" - } - } - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [ - "ai", - "vllm", - "dcgm", - "gpu", - "snarfnet" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m" - ] - }, - "timezone": "browser", - "title": "SnarfNet AI Stack & Inference Intelligence", - "uid": "snarfnet-ai-stack", - "version": 1 - } + ai-stack-dashboard.json: "{\n \"annotations\": {\n \"list\": [\n {\n \ + \ \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"\ + enable\": true,\n \"hide\": true,\n \"name\": \"Annotations & Alerts\"\ + ,\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n\ + \ \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 1,\n \"id\": null,\n \"\ + links\": [],\n \"liveNow\": false,\n \"panels\": [\n {\n \"collapsed\"\ + : false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \ + \ \"x\": 0,\n \"y\": 0\n },\n \"id\": 100,\n \"title\"\ + : \"\\u26a1 AI Stack Executive Overview & Cloud Cost Offsets\",\n \"type\"\ + : \"row\"\n },\n {\n \"id\": 1,\n \"title\": \"Prompt Tokens Processed\ + \ (In)\",\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n\ + \ \"w\": 6,\n \"x\": 0,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"sum(vllm:prompt_tokens_total)\"\ + ,\n \"legendFormat\": \"Prompt Tokens\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"thresholds\"\n },\n \ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \"\ + steps\": [\n {\n \"color\": \"blue\",\n \ + \ \"value\": null\n }\n ]\n },\n \ + \ \"unit\": \"locale\"\n }\n },\n \"options\": {\n \"\ + colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\"\ + : \"center\"\n }\n },\n {\n \"id\": 2,\n \"title\": \"Generation\ + \ Tokens Synthesized (Out)\",\n \"type\": \"stat\",\n \"gridPos\": {\n\ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 6,\n \"y\": 1\n \ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"sum(vllm:generation_tokens_total)\",\n \"\ + legendFormat\": \"Gen Tokens\",\n \"refId\": \"A\"\n }\n \ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\":\ + \ {\n \"mode\": \"thresholds\"\n },\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\": null\n\ + \ }\n ]\n },\n \"unit\": \"locale\"\n\ + \ }\n },\n \"options\": {\n \"colorMode\": \"value\",\n\ + \ \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 3,\n \"title\": \"Homelab Cloud API Savings\ + \ ($USD)\",\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n\ + \ \"w\": 6,\n \"x\": 12,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"(sum(vllm:prompt_tokens_total)\ + \ * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)\",\n \ + \ \"legendFormat\": \"Net Savings\",\n \"refId\": \"A\"\n }\n\ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"thresholds\"\n },\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\": null\n\ + \ }\n ]\n },\n \"unit\": \"currencyUSD\"\ + \n }\n },\n \"options\": {\n \"colorMode\": \"background\"\ + ,\n \"graphMode\": \"none\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 4,\n \"title\": \"Active GPU Cluster\ + \ Health\",\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n\ + \ \"w\": 6,\n \"x\": 18,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"count(DCGM_FI_DEV_GPU_TEMP)\"\ + ,\n \"legendFormat\": \"Tesla T4s Active\",\n \"refId\": \"\ + A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"purple\",\n \ + \ \"value\": null\n }\n ]\n },\n\ + \ \"unit\": \"short\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"none\",\n \"\ + justifyMode\": \"center\"\n }\n },\n {\n \"collapsed\": false,\n\ + \ \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n\ + \ \"y\": 5\n },\n \"id\": 200,\n \"title\": \"\\ud83c\\\ + udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)\",\n \"type\"\ + : \"row\"\n },\n {\n \"id\": 5,\n \"title\": \"GPU Core Utilization\ + \ (%)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\"\ + : 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 6\n },\n \ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"DCGM_FI_DEV_GPU_UTIL\",\n \"legendFormat\": \"{{Hostname}}\ + \ ({{modelName}})\",\n \"refId\": \"A\"\n }\n ],\n \"\ + fieldConfig\": {\n \"defaults\": {\n \"custom\": {\n \ + \ \"drawStyle\": \"line\",\n \"lineInterpolation\": \"smooth\",\n\ + \ \"fillOpacity\": 20\n },\n \"unit\": \"percent\"\ + ,\n \"min\": 0,\n \"max\": 100\n }\n }\n },\n\ + \ {\n \"id\": 6,\n \"title\": \"VRAM Memory Allocated & Used\",\n\ + \ \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \ + \ \"w\": 12,\n \"x\": 12,\n \"y\": 6\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"DCGM_FI_DEV_FB_USED\ + \ * 1024 * 1024\",\n \"legendFormat\": \"{{Hostname}} VRAM Used\",\n\ + \ \"refId\": \"A\"\n },\n {\n \"expr\": \"(DCGM_FI_DEV_FB_USED\ + \ + DCGM_FI_DEV_FB_FREE) * 1024 * 1024\",\n \"legendFormat\": \"{{Hostname}}\ + \ VRAM Total\",\n \"refId\": \"B\"\n }\n ],\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"custom\": {\n \"drawStyle\"\ + : \"line\",\n \"lineInterpolation\": \"smooth\",\n \"fillOpacity\"\ + : 15\n },\n \"unit\": \"bytes\"\n }\n }\n },\n\ + \ {\n \"id\": 7,\n \"title\": \"GPU Thermal Profile (\\u00b0C)\"\ + ,\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 7,\n\ + \ \"w\": 12,\n \"x\": 0,\n \"y\": 14\n },\n \"\ + datasource\": \"Prometheus\",\n \"targets\": [\n {\n \"expr\"\ + : \"DCGM_FI_DEV_GPU_TEMP\",\n \"legendFormat\": \"{{Hostname}} (\\u00b0C)\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"custom\": {\n \"drawStyle\":\ + \ \"line\",\n \"lineInterpolation\": \"smooth\",\n \"fillOpacity\"\ + : 10\n },\n \"unit\": \"celsius\",\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\": null\n\ + \ },\n {\n \"color\": \"yellow\",\n \ + \ \"value\": 65\n },\n {\n \ + \ \"color\": \"red\",\n \"value\": 80\n }\n \ + \ ]\n }\n }\n }\n },\n {\n \"id\": 8,\n\ + \ \"title\": \"GPU Power Draw (Watts)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 14\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"DCGM_FI_DEV_POWER_USAGE\"\ + ,\n \"legendFormat\": \"{{Hostname}} (Watts)\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"fillOpacity\": 15\n \ + \ },\n \"unit\": \"watt\"\n }\n }\n },\n {\n \ + \ \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \ + \ \"w\": 24,\n \"x\": 0,\n \"y\": 21\n },\n \"id\": 300,\n\ + \ \"title\": \"\\ud83e\\udde0 vLLM Real-Time Inference Dynamics & Token Velocity\"\ + ,\n \"type\": \"row\"\n },\n {\n \"id\": 9,\n \"title\":\ + \ \"Token Generation Velocity (Tokens / Sec)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 0,\n \"y\": 22\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum(rate(vllm:generation_tokens_total[2m]))\ + \ by (model_name)\",\n \"legendFormat\": \"{{model_name}} (Out)\",\n\ + \ \"refId\": \"A\"\n },\n {\n \"expr\": \"sum(rate(vllm:prompt_tokens_total[2m]))\ + \ by (model_name)\",\n \"legendFormat\": \"{{model_name}} (In)\",\n \ + \ \"refId\": \"B\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"custom\": {\n \"drawStyle\": \"\ + line\",\n \"lineInterpolation\": \"smooth\",\n \"fillOpacity\"\ + : 25\n },\n \"unit\": \"cps\"\n }\n }\n },\n\ + \ {\n \"id\": 10,\n \"title\": \"Inference Concurrency & Request\ + \ Backlog\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"\ + h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 22\n },\n\ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"vllm:num_requests_running\",\n \"legendFormat\": \"\ + {{model_name}} Running\",\n \"refId\": \"A\"\n },\n {\n\ + \ \"expr\": \"vllm:num_requests_waiting\",\n \"legendFormat\"\ + : \"{{model_name}} Queued\",\n \"refId\": \"B\"\n }\n ],\n\ + \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {\n\ + \ \"drawStyle\": \"line\",\n \"lineInterpolation\": \"stepAfter\"\ + ,\n \"fillOpacity\": 15\n },\n \"unit\": \"short\"\ + \n }\n }\n },\n {\n \"id\": 11,\n \"title\": \"GPU\ + \ KV Cache Memory Saturation\",\n \"type\": \"timeseries\",\n \"gridPos\"\ + : {\n \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\":\ + \ 30\n },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n\ + \ {\n \"expr\": \"vllm:gpu_cache_usage_factor * 100\",\n \ + \ \"legendFormat\": \"{{model_name}} Cache Usage\",\n \"refId\":\ + \ \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"fillOpacity\": 30\n \ + \ },\n \"unit\": \"percent\",\n \"min\": 0,\n \ + \ \"max\": 100\n }\n }\n },\n {\n \"id\": 12,\n \"\ + title\": \"NVIDIA Triton & Qdrant Operations\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 30\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"rate(nv_inference_request_success[2m])\"\ + ,\n \"legendFormat\": \"Triton Success/s\",\n \"refId\": \"\ + A\"\n },\n {\n \"expr\": \"cpu_cores_used\",\n \ + \ \"legendFormat\": \"Qdrant CPU Cores\",\n \"refId\": \"B\"\n \ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineInterpolation\"\ + : \"smooth\"\n },\n \"unit\": \"short\"\n }\n }\n\ + \ }\n ],\n \"refresh\": \"10s\",\n \"schemaVersion\": 39,\n \"tags\": [\n\ + \ \"ai\",\n \"vllm\",\n \"dcgm\",\n \"gpu\",\n \"snarfnet\"\n \ + \ ],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\"\ + : \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\"\ + : [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\"\ + \n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"SnarfNet AI Stack\ + \ & Inference Intelligence\",\n \"uid\": \"snarfnet-ai-stack\",\n \"version\"\ + : 1\n}\n" --- apiVersion: v1 kind: ConfigMap metadata: - name: grafana-dashboard-systems + name: grafana-dashboard-aerodrome namespace: default labels: app.kubernetes.io/name: grafana app.kubernetes.io/instance: grafana data: - systems-dashboard.json: |- - { - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "title": "\ud83d\udda5\ufe0f Systems Fleet Overview & Capacity", - "type": "row" - }, - { - "id": 1, - "title": "Online Node Exporters", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "count(node_load1)", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 2, - "title": "Avg Cluster CPU Load (1m)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "avg(node_load1)", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 3, - "title": "Total Monitored RAM", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(node_memory_MemTotal_bytes)", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "bytes" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 4, - "title": "Fleet RAM In-Use %", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - }, - "unit": "percent" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 200, - "title": "\u26a1 CPU Load & Core Utilization", - "type": "row" - }, - { - "id": 5, - "title": "CPU Utilization by Host (%)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)", - "legendFormat": "{{cluster}} - {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "percent" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 6, - "title": "1-Minute Load Average by Host", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "node_load1", - "legendFormat": "{{cluster}} - {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "none" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 300, - "title": "\ud83e\udde0 Memory Consumption & Free Capacity", - "type": "row" - }, - { - "id": 7, - "title": "Memory Used % by Host", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", - "legendFormat": "{{cluster}} - {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "percent" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 8, - "title": "Available RAM by Host (GiB)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "node_memory_MemAvailable_bytes", - "legendFormat": "{{cluster}} - {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "bytes" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 400, - "title": "\ud83d\udcbe Filesystem & Disk I/O Performance", - "type": "row" - }, - { - "id": 9, - "title": "Root (/) Filesystem Usage %", - "type": "bargauge", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100 - } - }, - "options": { - "displayMode": "gradient", - "orientation": "horizontal", - "showUnfilled": true - } - }, - { - "id": 10, - "title": "Disk Read Throughput (Bytes/sec)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 24 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))", - "legendFormat": "Read {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "Bps" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 11, - "title": "Disk Write Throughput (Bytes/sec)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 24 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))", - "legendFormat": "Write {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "Bps" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 500, - "title": "\ud83c\udf10 Network Throughput (RX / TX)", - "type": "row" - }, - { - "id": 12, - "title": "Host Network Receive (RX)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", - "legendFormat": "RX {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "Bps" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 13, - "title": "Host Network Transmit (TX)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", - "legendFormat": "TX {{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "Bps" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [ - "infrastructure", - "systems", - "snarfnet", - "node-exporter" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "\ud83d\udda5\ufe0f Systems & Cluster Host Performance", - "uid": "snarfnet-systems" - } + aerodrome-resource-monitor.json: "{\n \"annotations\": {\n \"list\": []\n },\n\ + \ \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 1,\n\ + \ \"id\": null,\n \"links\": [],\n \"liveNow\": false,\n \"refresh\": \"15s\"\ + ,\n \"schemaVersion\": 39,\n \"tags\": [\n \"aerodrome\",\n \"resource-monitor\"\ + ,\n \"infrastructure\"\n ],\n \"time\": {\n \"from\": \"now-1h\",\n \ + \ \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n\ + \ \"5s\",\n \"10s\",\n \"15s\",\n \"30s\",\n \"1m\",\n\ + \ \"5m\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"\\u2708\\\ + ufe0f Aerodrome Resource Monitor\",\n \"uid\": \"aerodrome-resource-monitor\"\ + ,\n \"version\": 1,\n \"panels\": [\n {\n \"collapsed\": false,\n \ + \ \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n\ + \ \"y\": 0\n },\n \"id\": 100,\n \"title\": \"\\ud83d\\\ + udda5\\ufe0f Systems & Host Performance \\u2014 Aerodrome Cluster Fleet\",\n \ + \ \"type\": \"row\"\n },\n {\n \"id\": 1,\n \"title\": \"\ + Online Nodes (Aerodrome)\",\n \"type\": \"stat\",\n \"gridPos\": {\n\ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 0,\n \"y\": 1\n \ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"count(node_load1{cluster=\\\"aerodrome\\\"})\",\n\ + \ \"legendFormat\": \"Nodes\",\n \"refId\": \"A\"\n }\n\ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"thresholds\"\n },\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\": null\n\ + \ }\n ]\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"colorMode\": \"value\",\n\ + \ \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 2,\n \"title\": \"Avg Cluster CPU Load\ + \ (1m)\",\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n\ + \ \"w\": 6,\n \"x\": 6,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"avg(node_load1{cluster=\\\ + \"aerodrome\\\"})\",\n \"legendFormat\": \"Load\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"blue\",\n \ + \ \"value\": null\n },\n {\n \ + \ \"color\": \"orange\",\n \"value\": 2.0\n },\n\ + \ {\n \"color\": \"red\",\n \"value\"\ + : 4.0\n }\n ]\n },\n \"unit\": \"none\"\ + \n }\n },\n \"options\": {\n \"colorMode\": \"value\"\ + ,\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 3,\n \"title\": \"Total Monitored RAM\"\ + ,\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n \ + \ \"w\": 6,\n \"x\": 12,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"sum(node_memory_MemTotal_bytes{cluster=\\\ + \"aerodrome\\\"})\",\n \"legendFormat\": \"RAM\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"purple\",\n \ + \ \"value\": null\n }\n ]\n },\n\ + \ \"unit\": \"bytes\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"\ + justifyMode\": \"center\"\n }\n },\n {\n \"id\": 4,\n \"\ + title\": \"Cluster RAM In-Use %\",\n \"type\": \"stat\",\n \"gridPos\"\ + : {\n \"h\": 4,\n \"w\": 6,\n \"x\": 18,\n \"y\":\ + \ 1\n },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"(1 - (sum(node_memory_MemAvailable_bytes{cluster=\\\ + \"aerodrome\\\"}) / sum(node_memory_MemTotal_bytes{cluster=\\\"aerodrome\\\"})))\ + \ * 100\",\n \"legendFormat\": \"In-Use\",\n \"refId\": \"A\"\ + \n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"thresholds\"\n },\n \ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \"\ + steps\": [\n {\n \"color\": \"green\",\n \ + \ \"value\": null\n },\n {\n \"\ + color\": \"orange\",\n \"value\": 75\n },\n \ + \ {\n \"color\": \"red\",\n \"value\": 90\n\ + \ }\n ]\n },\n \"unit\": \"percent\"\ + \n }\n },\n \"options\": {\n \"colorMode\": \"value\"\ + ,\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 5,\n \"title\": \"CPU Utilization by\ + \ Host (%)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 5\n },\n\ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\\\ + \"idle\\\", cluster=\\\"aerodrome\\\"}[2m])) * 100)\",\n \"legendFormat\"\ + : \"{{instance}}\",\n \"refId\": \"A\"\n }\n ],\n \"\ + fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \ + \ \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \ + \ \"drawStyle\": \"line\",\n \"lineInterpolation\": \"smooth\"\ + ,\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n },\n\ + \ \"unit\": \"percent\",\n \"min\": 0,\n \"max\": 100\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 6,\n \ + \ \"title\": \"1-Minute Load Average by Host\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 5\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"node_load1{cluster=\\\"aerodrome\\\ + \"}\",\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"\ + A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"none\",\n \ + \ \"min\": 0\n }\n },\n \"options\": {\n \"legend\"\ + : {\n \"displayMode\": \"table\",\n \"placement\": \"bottom\"\ + ,\n \"calcs\": [\n \"mean\",\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n }\n }\n },\n {\n \ + \ \"id\": 7,\n \"title\": \"Memory Used % by Host\",\n \"type\": \"\ + timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \ + \ \"x\": 0,\n \"y\": 13\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"(1 - (node_memory_MemAvailable_bytes{cluster=\\\ + \"aerodrome\\\"} / node_memory_MemTotal_bytes{cluster=\\\"aerodrome\\\"})) * 100\"\ + ,\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n\ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"\ + lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \"\ + fillOpacity\": 15\n },\n \"unit\": \"percent\",\n \"\ + min\": 0,\n \"max\": 100\n }\n },\n \"options\": {\n\ + \ \"legend\": {\n \"displayMode\": \"table\",\n \"placement\"\ + : \"bottom\",\n \"calcs\": [\n \"mean\",\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n }\n }\n },\n {\n \ + \ \"id\": 8,\n \"title\": \"Available RAM by Host (GiB)\",\n \"type\"\ + : \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n\ + \ \"x\": 12,\n \"y\": 13\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"node_memory_MemAvailable_bytes{cluster=\\\ + \"aerodrome\\\"}\",\n \"legendFormat\": \"{{instance}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"\ + defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 10\n },\n \"unit\": \"bytes\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + min\"\n ]\n }\n }\n },\n {\n \"id\": 9,\n \ + \ \"title\": \"Root (/) Filesystem Usage %\",\n \"type\": \"bargauge\",\n\ + \ \"gridPos\": {\n \"h\": 8,\n \"w\": 8,\n \"x\": 0,\n\ + \ \"y\": 21\n },\n \"datasource\": \"Prometheus\",\n \"\ + targets\": [\n {\n \"expr\": \"(1 - (node_filesystem_avail_bytes{mountpoint=\\\ + \"/\\\", cluster=\\\"aerodrome\\\"} / node_filesystem_size_bytes{mountpoint=\\\ + \"/\\\", cluster=\\\"aerodrome\\\"})) * 100\",\n \"legendFormat\": \"\ + {{instance}}\",\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"color\": {\n \"mode\": \"\ + thresholds\"\n },\n \"thresholds\": {\n \"mode\"\ + : \"absolute\",\n \"steps\": [\n {\n \"\ + color\": \"green\",\n \"value\": null\n },\n \ + \ {\n \"color\": \"orange\",\n \"value\"\ + : 75\n },\n {\n \"color\": \"red\",\n\ + \ \"value\": 90\n }\n ]\n },\n\ + \ \"unit\": \"percent\",\n \"min\": 0,\n \"max\": 100\n\ + \ }\n },\n \"options\": {\n \"displayMode\": \"gradient\"\ + ,\n \"orientation\": \"horizontal\",\n \"showUnfilled\": true\n\ + \ }\n },\n {\n \"id\": 10,\n \"title\": \"Disk Read Throughput\"\ + ,\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n\ + \ \"w\": 8,\n \"x\": 8,\n \"y\": 21\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"sum\ + \ by (instance) (rate(node_disk_read_bytes_total{cluster=\\\"aerodrome\\\"}[2m]))\"\ + ,\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n\ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"\ + lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \"\ + fillOpacity\": 10\n },\n \"unit\": \"Bps\"\n }\n \ + \ },\n \"options\": {\n \"legend\": {\n \"displayMode\"\ + : \"table\",\n \"placement\": \"bottom\",\n \"calcs\": [\n \ + \ \"mean\",\n \"max\"\n ]\n }\n }\n\ + \ },\n {\n \"id\": 11,\n \"title\": \"Disk Write Throughput\"\ + ,\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n\ + \ \"w\": 8,\n \"x\": 16,\n \"y\": 21\n },\n \"\ + datasource\": \"Prometheus\",\n \"targets\": [\n {\n \"expr\"\ + : \"sum by (instance) (rate(node_disk_written_bytes_total{cluster=\\\"aerodrome\\\ + \"}[2m]))\",\n \"legendFormat\": \"{{instance}}\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"Bps\"\n \ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n },\n {\n \"id\": 12,\n \"title\": \"Host Network Receive\ + \ (RX)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\"\ + : 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 29\n },\n \ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"sum by (instance) (rate(node_network_receive_bytes_total{device!~\\\ + \"lo|veth.*|flannel.*|cali.*\\\", cluster=\\\"aerodrome\\\"}[2m]))\",\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n \ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineInterpolation\"\ + : \"smooth\",\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n\ + \ },\n \"unit\": \"Bps\"\n }\n },\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"mean\",\n \ + \ \"max\"\n ]\n }\n }\n },\n {\n \"id\"\ + : 13,\n \"title\": \"Host Network Transmit (TX)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 29\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (instance) (rate(node_network_transmit_bytes_total{device!~\\\ + \"lo|veth.*|flannel.*|cali.*\\\", cluster=\\\"aerodrome\\\"}[2m]))\",\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n \ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineInterpolation\"\ + : \"smooth\",\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n\ + \ },\n \"unit\": \"Bps\"\n }\n },\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"mean\",\n \ + \ \"max\"\n ]\n }\n }\n },\n {\n \"collapsed\"\ + : false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \ + \ \"x\": 0,\n \"y\": 37\n },\n \"id\": 200,\n \"title\"\ + : \"\\ud83d\\udce6 Kubernetes Containers & Workloads \\u2014 Aerodrome Cluster\"\ + ,\n \"type\": \"row\"\n },\n {\n \"id\": 14,\n \"title\"\ + : \"Monitored Containers\",\n \"type\": \"stat\",\n \"gridPos\": {\n\ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 0,\n \"y\": 38\n\ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"count(container_last_seen{container!=\\\"\\\", instance=~\\\ + \"sninfr2.*\\\"})\",\n \"legendFormat\": \"Containers\",\n \"\ + refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"green\",\n \ + \ \"value\": null\n }\n ]\n },\n \ + \ \"unit\": \"none\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\"\ + : \"center\"\n }\n },\n {\n \"id\": 15,\n \"title\": \"Active\ + \ Kubernetes Pods\",\n \"type\": \"stat\",\n \"gridPos\": {\n \ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 6,\n \"y\": 38\n },\n\ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"count(count by (pod, namespace) (container_last_seen{container!=\\\ + \"\\\", instance=~\\\"sninfr2.*\\\"}))\",\n \"legendFormat\": \"Pods\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\ + \n },\n \"thresholds\": {\n \"mode\": \"absolute\"\ + ,\n \"steps\": [\n {\n \"color\": \"blue\"\ + ,\n \"value\": null\n }\n ]\n \ + \ },\n \"unit\": \"none\"\n }\n },\n \"options\": {\n\ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \ + \ \"justifyMode\": \"center\"\n }\n },\n {\n \"id\": 16,\n \ + \ \"title\": \"Total Container CPU Load\",\n \"type\": \"stat\",\n \ + \ \"gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \"x\": 12,\n\ + \ \"y\": 38\n },\n \"datasource\": \"Prometheus\",\n \"\ + targets\": [\n {\n \"expr\": \"sum(rate(container_cpu_usage_seconds_total{container!=\\\ + \"\\\", instance=~\\\"sninfr2.*\\\"}[2m]))\",\n \"legendFormat\": \"\ + Cores\",\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"color\": {\n \"mode\": \"\ + thresholds\"\n },\n \"thresholds\": {\n \"mode\"\ + : \"absolute\",\n \"steps\": [\n {\n \"\ + color\": \"orange\",\n \"value\": null\n }\n \ + \ ]\n },\n \"unit\": \"none\"\n }\n },\n \ + \ \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\"\ + : \"area\",\n \"justifyMode\": \"center\"\n }\n },\n {\n \ + \ \"id\": 17,\n \"title\": \"Total Container Working RAM\",\n \"type\"\ + : \"stat\",\n \"gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \ + \ \"x\": 18,\n \"y\": 38\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"sum(container_memory_working_set_bytes{container!=\\\ + \"\\\", instance=~\\\"sninfr2.*\\\"})\",\n \"legendFormat\": \"RAM\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\ + \n },\n \"thresholds\": {\n \"mode\": \"absolute\"\ + ,\n \"steps\": [\n {\n \"color\": \"purple\"\ + ,\n \"value\": null\n }\n ]\n \ + \ },\n \"unit\": \"bytes\"\n }\n },\n \"options\": {\n\ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \ + \ \"justifyMode\": \"center\"\n }\n },\n {\n \"id\": 18,\n \ + \ \"title\": \"Top 10 CPU Consuming Pods (Cores)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 0,\n \"y\": 42\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (pod, namespace)\ + \ (rate(container_cpu_usage_seconds_total{container!=\\\"\\\", instance=~\\\"\ + sninfr2.*\\\"}[2m])))\",\n \"legendFormat\": \"{{namespace}}/{{pod}}\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 15\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 19,\n \ + \ \"title\": \"CPU Throttling by Namespace\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 42\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{instance=~\\\ + \"sninfr2.*\\\"}[2m]))\",\n \"legendFormat\": \"{{namespace}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 10\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 20,\n \ + \ \"title\": \"Top 10 Memory Consuming Pods\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 14,\n \"x\"\ + : 0,\n \"y\": 50\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (pod, namespace)\ + \ (container_memory_working_set_bytes{container!=\\\"\\\", instance=~\\\"sninfr2.*\\\ + \"}))\",\n \"legendFormat\": \"{{namespace}}/{{pod}}\",\n \"\ + refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 15\n },\n \"unit\": \"bytes\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 21,\n \ + \ \"title\": \"Memory Working Set by Namespace\",\n \"type\": \"bargauge\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 10,\n \"x\"\ + : 14,\n \"y\": 50\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (namespace) (container_memory_working_set_bytes{container!=\\\ + \"\\\", instance=~\\\"sninfr2.*\\\"})\",\n \"legendFormat\": \"{{namespace}}\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"unit\": \"bytes\"\n }\n },\n \"options\"\ + : {\n \"displayMode\": \"gradient\",\n \"orientation\": \"horizontal\"\ + ,\n \"showUnfilled\": true\n }\n },\n {\n \"id\": 22,\n\ + \ \"title\": \"Top Pod Network RX (Receive)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 0,\n \"y\": 58\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (pod, namespace)\ + \ (rate(container_network_receive_bytes_total{instance=~\\\"sninfr2.*\\\"}[2m])))\"\ + ,\n \"legendFormat\": \"{{namespace}}/{{pod}}\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 15\n },\n \"unit\": \"Bps\"\n \ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n },\n {\n \"id\": 23,\n \"title\": \"Top Pod Network\ + \ TX (Transmit)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 58\n\ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{instance=~\\\ + \"sninfr2.*\\\"}[2m])))\",\n \"legendFormat\": \"{{namespace}}/{{pod}}\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 15\n },\n \"unit\": \"Bps\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n }\n ]\n}" --- apiVersion: v1 kind: ConfigMap metadata: - name: grafana-dashboard-containers + name: grafana-dashboard-spaceport namespace: default labels: app.kubernetes.io/name: grafana app.kubernetes.io/instance: grafana data: - container-metrics-dashboard.json: |- - { - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "title": "\ud83d\udce6 Container Fleet & Pod Metrics", - "type": "row" - }, - { - "id": 1, - "title": "Monitored Containers", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "count(container_last_seen{container!=''})", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 2, - "title": "Active Kubernetes Pods", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "count(count by (pod, namespace) (container_last_seen{container!=''}))", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 3, - "title": "Total Container CPU Load", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "none" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "id": 4, - "title": "Total Container RAM", - "type": "stat", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{container!=''})", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - }, - "unit": "bytes" - } - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center" - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 200, - "title": "\u26a1 Top Container CPU Utilization", - "type": "row" - }, - { - "id": 5, - "title": "Top 10 CPU Consuming Pods (Cores)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))", - "legendFormat": "{{namespace}} / {{pod}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "none" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 6, - "title": "CPU Throttling by Namespace", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "none" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 300, - "title": "\ud83e\udde0 Container Memory Working Set", - "type": "row" - }, - { - "id": 7, - "title": "Top 10 Memory Consuming Pods", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 14, - "x": 0, - "y": 15 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))", - "legendFormat": "{{namespace}} / {{pod}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "bytes" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 8, - "title": "Memory by Namespace", - "type": "bargauge", - "gridPos": { - "h": 8, - "w": 10, - "x": 14, - "y": 15 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=''})", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "bytes", - "min": 0, - "max": 100 - } - }, - "options": { - "displayMode": "gradient", - "orientation": "horizontal", - "showUnfilled": true - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 400, - "title": "\ud83c\udf10 Container Network Traffic", - "type": "row" - }, - { - "id": 9, - "title": "Top Pod Network RX (Receive)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))", - "legendFormat": "RX {{namespace}}/{{pod}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "Bps" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 10, - "title": "Top Pod Network TX (Transmit)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "datasource": "Prometheus", - "targets": [ - { - "expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))", - "legendFormat": "TX {{namespace}}/{{pod}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15 - }, - "unit": "Bps" - } - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [ - "kubernetes", - "containers", - "cadvisor", - "snarfnet", - "workloads" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "\ud83d\udce6 Kubernetes Containers & Workloads", - "uid": "snarfnet-containers" - } + spaceport-resource-monitor.json: "{\n \"annotations\": {\n \"list\": []\n },\n\ + \ \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 1,\n\ + \ \"id\": null,\n \"links\": [],\n \"liveNow\": false,\n \"refresh\": \"15s\"\ + ,\n \"schemaVersion\": 39,\n \"tags\": [\n \"spaceport\",\n \"resource-monitor\"\ + ,\n \"infrastructure\"\n ],\n \"time\": {\n \"from\": \"now-1h\",\n \ + \ \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n\ + \ \"5s\",\n \"10s\",\n \"15s\",\n \"30s\",\n \"1m\",\n\ + \ \"5m\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"\\ud83d\\\ + ude80 Spaceport Resource Monitor\",\n \"uid\": \"spaceport-resource-monitor\"\ + ,\n \"version\": 1,\n \"panels\": [\n {\n \"collapsed\": false,\n \ + \ \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n\ + \ \"y\": 0\n },\n \"id\": 100,\n \"title\": \"\\ud83d\\\ + udda5\\ufe0f Systems & Host Performance \\u2014 Spaceport Cluster Fleet\",\n \ + \ \"type\": \"row\"\n },\n {\n \"id\": 1,\n \"title\": \"\ + Online Nodes (Spaceport)\",\n \"type\": \"stat\",\n \"gridPos\": {\n\ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 0,\n \"y\": 1\n \ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"count(node_load1{cluster=\\\"spaceport\\\"})\",\n\ + \ \"legendFormat\": \"Nodes\",\n \"refId\": \"A\"\n }\n\ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"thresholds\"\n },\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\": null\n\ + \ }\n ]\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"colorMode\": \"value\",\n\ + \ \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 2,\n \"title\": \"Avg Cluster CPU Load\ + \ (1m)\",\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n\ + \ \"w\": 6,\n \"x\": 6,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"avg(node_load1{cluster=\\\ + \"spaceport\\\"})\",\n \"legendFormat\": \"Load\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"blue\",\n \ + \ \"value\": null\n },\n {\n \ + \ \"color\": \"orange\",\n \"value\": 2.0\n },\n\ + \ {\n \"color\": \"red\",\n \"value\"\ + : 4.0\n }\n ]\n },\n \"unit\": \"none\"\ + \n }\n },\n \"options\": {\n \"colorMode\": \"value\"\ + ,\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 3,\n \"title\": \"Total Monitored RAM\"\ + ,\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n \ + \ \"w\": 6,\n \"x\": 12,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"sum(node_memory_MemTotal_bytes{cluster=\\\ + \"spaceport\\\"})\",\n \"legendFormat\": \"RAM\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"purple\",\n \ + \ \"value\": null\n }\n ]\n },\n\ + \ \"unit\": \"bytes\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"\ + justifyMode\": \"center\"\n }\n },\n {\n \"id\": 4,\n \"\ + title\": \"Cluster RAM In-Use %\",\n \"type\": \"stat\",\n \"gridPos\"\ + : {\n \"h\": 4,\n \"w\": 6,\n \"x\": 18,\n \"y\":\ + \ 1\n },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"(1 - (sum(node_memory_MemAvailable_bytes{cluster=\\\ + \"spaceport\\\"}) / sum(node_memory_MemTotal_bytes{cluster=\\\"spaceport\\\"})))\ + \ * 100\",\n \"legendFormat\": \"In-Use\",\n \"refId\": \"A\"\ + \n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"thresholds\"\n },\n \ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \"\ + steps\": [\n {\n \"color\": \"green\",\n \ + \ \"value\": null\n },\n {\n \"\ + color\": \"orange\",\n \"value\": 75\n },\n \ + \ {\n \"color\": \"red\",\n \"value\": 90\n\ + \ }\n ]\n },\n \"unit\": \"percent\"\ + \n }\n },\n \"options\": {\n \"colorMode\": \"value\"\ + ,\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 5,\n \"title\": \"CPU Utilization by\ + \ Host (%)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 5\n },\n\ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\\\ + \"idle\\\", cluster=\\\"spaceport\\\"}[2m])) * 100)\",\n \"legendFormat\"\ + : \"{{instance}}\",\n \"refId\": \"A\"\n }\n ],\n \"\ + fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \ + \ \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \ + \ \"drawStyle\": \"line\",\n \"lineInterpolation\": \"smooth\"\ + ,\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n },\n\ + \ \"unit\": \"percent\",\n \"min\": 0,\n \"max\": 100\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 6,\n \ + \ \"title\": \"1-Minute Load Average by Host\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 5\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"node_load1{cluster=\\\"spaceport\\\ + \"}\",\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"\ + A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"none\",\n \ + \ \"min\": 0\n }\n },\n \"options\": {\n \"legend\"\ + : {\n \"displayMode\": \"table\",\n \"placement\": \"bottom\"\ + ,\n \"calcs\": [\n \"mean\",\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n }\n }\n },\n {\n \ + \ \"id\": 7,\n \"title\": \"Memory Used % by Host\",\n \"type\": \"\ + timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \ + \ \"x\": 0,\n \"y\": 13\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"(1 - (node_memory_MemAvailable_bytes{cluster=\\\ + \"spaceport\\\"} / node_memory_MemTotal_bytes{cluster=\\\"spaceport\\\"})) * 100\"\ + ,\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n\ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"\ + lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \"\ + fillOpacity\": 15\n },\n \"unit\": \"percent\",\n \"\ + min\": 0,\n \"max\": 100\n }\n },\n \"options\": {\n\ + \ \"legend\": {\n \"displayMode\": \"table\",\n \"placement\"\ + : \"bottom\",\n \"calcs\": [\n \"mean\",\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n }\n }\n },\n {\n \ + \ \"id\": 8,\n \"title\": \"Available RAM by Host (GiB)\",\n \"type\"\ + : \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n\ + \ \"x\": 12,\n \"y\": 13\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"node_memory_MemAvailable_bytes{cluster=\\\ + \"spaceport\\\"}\",\n \"legendFormat\": \"{{instance}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"\ + defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 10\n },\n \"unit\": \"bytes\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + min\"\n ]\n }\n }\n },\n {\n \"id\": 9,\n \ + \ \"title\": \"Root (/) Filesystem Usage %\",\n \"type\": \"bargauge\",\n\ + \ \"gridPos\": {\n \"h\": 8,\n \"w\": 8,\n \"x\": 0,\n\ + \ \"y\": 21\n },\n \"datasource\": \"Prometheus\",\n \"\ + targets\": [\n {\n \"expr\": \"(1 - (node_filesystem_avail_bytes{mountpoint=\\\ + \"/\\\", cluster=\\\"spaceport\\\"} / node_filesystem_size_bytes{mountpoint=\\\ + \"/\\\", cluster=\\\"spaceport\\\"})) * 100\",\n \"legendFormat\": \"\ + {{instance}}\",\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"color\": {\n \"mode\": \"\ + thresholds\"\n },\n \"thresholds\": {\n \"mode\"\ + : \"absolute\",\n \"steps\": [\n {\n \"\ + color\": \"green\",\n \"value\": null\n },\n \ + \ {\n \"color\": \"orange\",\n \"value\"\ + : 75\n },\n {\n \"color\": \"red\",\n\ + \ \"value\": 90\n }\n ]\n },\n\ + \ \"unit\": \"percent\",\n \"min\": 0,\n \"max\": 100\n\ + \ }\n },\n \"options\": {\n \"displayMode\": \"gradient\"\ + ,\n \"orientation\": \"horizontal\",\n \"showUnfilled\": true\n\ + \ }\n },\n {\n \"id\": 10,\n \"title\": \"Disk Read Throughput\"\ + ,\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n\ + \ \"w\": 8,\n \"x\": 8,\n \"y\": 21\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"sum\ + \ by (instance) (rate(node_disk_read_bytes_total{cluster=\\\"spaceport\\\"}[2m]))\"\ + ,\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n\ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"\ + lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \"\ + fillOpacity\": 10\n },\n \"unit\": \"Bps\"\n }\n \ + \ },\n \"options\": {\n \"legend\": {\n \"displayMode\"\ + : \"table\",\n \"placement\": \"bottom\",\n \"calcs\": [\n \ + \ \"mean\",\n \"max\"\n ]\n }\n }\n\ + \ },\n {\n \"id\": 11,\n \"title\": \"Disk Write Throughput\"\ + ,\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n\ + \ \"w\": 8,\n \"x\": 16,\n \"y\": 21\n },\n \"\ + datasource\": \"Prometheus\",\n \"targets\": [\n {\n \"expr\"\ + : \"sum by (instance) (rate(node_disk_written_bytes_total{cluster=\\\"spaceport\\\ + \"}[2m]))\",\n \"legendFormat\": \"{{instance}}\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"Bps\"\n \ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n },\n {\n \"id\": 12,\n \"title\": \"Host Network Receive\ + \ (RX)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\"\ + : 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 29\n },\n \ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"sum by (instance) (rate(node_network_receive_bytes_total{device!~\\\ + \"lo|veth.*|flannel.*|cali.*\\\", cluster=\\\"spaceport\\\"}[2m]))\",\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n \ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineInterpolation\"\ + : \"smooth\",\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n\ + \ },\n \"unit\": \"Bps\"\n }\n },\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"mean\",\n \ + \ \"max\"\n ]\n }\n }\n },\n {\n \"id\"\ + : 13,\n \"title\": \"Host Network Transmit (TX)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 29\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (instance) (rate(node_network_transmit_bytes_total{device!~\\\ + \"lo|veth.*|flannel.*|cali.*\\\", cluster=\\\"spaceport\\\"}[2m]))\",\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n \ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineInterpolation\"\ + : \"smooth\",\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n\ + \ },\n \"unit\": \"Bps\"\n }\n },\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"mean\",\n \ + \ \"max\"\n ]\n }\n }\n },\n {\n \"collapsed\"\ + : false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \ + \ \"x\": 0,\n \"y\": 37\n },\n \"id\": 200,\n \"title\"\ + : \"\\ud83d\\udce6 Kubernetes Containers & Workloads \\u2014 Spaceport Cluster\"\ + ,\n \"type\": \"row\"\n },\n {\n \"id\": 14,\n \"title\"\ + : \"Monitored Containers\",\n \"type\": \"stat\",\n \"gridPos\": {\n\ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 0,\n \"y\": 38\n\ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"count(container_last_seen{container!=\\\"\\\", cluster=\\\ + \"spaceport\\\"})\",\n \"legendFormat\": \"Containers\",\n \"\ + refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"green\",\n \ + \ \"value\": null\n }\n ]\n },\n \ + \ \"unit\": \"none\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\"\ + : \"center\"\n }\n },\n {\n \"id\": 15,\n \"title\": \"Active\ + \ Kubernetes Pods\",\n \"type\": \"stat\",\n \"gridPos\": {\n \ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 6,\n \"y\": 38\n },\n\ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"count(count by (pod, namespace) (container_last_seen{container!=\\\ + \"\\\", cluster=\\\"spaceport\\\"}))\",\n \"legendFormat\": \"Pods\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\ + \n },\n \"thresholds\": {\n \"mode\": \"absolute\"\ + ,\n \"steps\": [\n {\n \"color\": \"blue\"\ + ,\n \"value\": null\n }\n ]\n \ + \ },\n \"unit\": \"none\"\n }\n },\n \"options\": {\n\ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \ + \ \"justifyMode\": \"center\"\n }\n },\n {\n \"id\": 16,\n \ + \ \"title\": \"Total Container CPU Load\",\n \"type\": \"stat\",\n \ + \ \"gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \"x\": 12,\n\ + \ \"y\": 38\n },\n \"datasource\": \"Prometheus\",\n \"\ + targets\": [\n {\n \"expr\": \"sum(rate(container_cpu_usage_seconds_total{container!=\\\ + \"\\\", cluster=\\\"spaceport\\\"}[2m]))\",\n \"legendFormat\": \"Cores\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\ + \n },\n \"thresholds\": {\n \"mode\": \"absolute\"\ + ,\n \"steps\": [\n {\n \"color\": \"orange\"\ + ,\n \"value\": null\n }\n ]\n \ + \ },\n \"unit\": \"none\"\n }\n },\n \"options\": {\n\ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \ + \ \"justifyMode\": \"center\"\n }\n },\n {\n \"id\": 17,\n \ + \ \"title\": \"Total Container Working RAM\",\n \"type\": \"stat\",\n \ + \ \"gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \"x\": 18,\n\ + \ \"y\": 38\n },\n \"datasource\": \"Prometheus\",\n \"\ + targets\": [\n {\n \"expr\": \"sum(container_memory_working_set_bytes{container!=\\\ + \"\\\", cluster=\\\"spaceport\\\"})\",\n \"legendFormat\": \"RAM\",\n\ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\ + \n },\n \"thresholds\": {\n \"mode\": \"absolute\"\ + ,\n \"steps\": [\n {\n \"color\": \"purple\"\ + ,\n \"value\": null\n }\n ]\n \ + \ },\n \"unit\": \"bytes\"\n }\n },\n \"options\": {\n\ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \ + \ \"justifyMode\": \"center\"\n }\n },\n {\n \"id\": 18,\n \ + \ \"title\": \"Top 10 CPU Consuming Pods (Cores)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 0,\n \"y\": 42\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (pod, namespace)\ + \ (rate(container_cpu_usage_seconds_total{container!=\\\"\\\", cluster=\\\"spaceport\\\ + \"}[2m])))\",\n \"legendFormat\": \"{{namespace}}/{{pod}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 15\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 19,\n \ + \ \"title\": \"CPU Throttling by Namespace\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 42\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{cluster=\\\ + \"spaceport\\\"}[2m]))\",\n \"legendFormat\": \"{{namespace}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 10\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 20,\n \ + \ \"title\": \"Top 10 Memory Consuming Pods\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 14,\n \"x\"\ + : 0,\n \"y\": 50\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (pod, namespace)\ + \ (container_memory_working_set_bytes{container!=\\\"\\\", cluster=\\\"spaceport\\\ + \"}))\",\n \"legendFormat\": \"{{namespace}}/{{pod}}\",\n \"\ + refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 15\n },\n \"unit\": \"bytes\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 21,\n \ + \ \"title\": \"Memory Working Set by Namespace\",\n \"type\": \"bargauge\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 10,\n \"x\"\ + : 14,\n \"y\": 50\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (namespace) (container_memory_working_set_bytes{container!=\\\ + \"\\\", cluster=\\\"spaceport\\\"})\",\n \"legendFormat\": \"{{namespace}}\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"unit\": \"bytes\"\n }\n },\n \"options\"\ + : {\n \"displayMode\": \"gradient\",\n \"orientation\": \"horizontal\"\ + ,\n \"showUnfilled\": true\n }\n },\n {\n \"id\": 22,\n\ + \ \"title\": \"Top Pod Network RX (Receive)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 0,\n \"y\": 58\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (pod, namespace)\ + \ (rate(container_network_receive_bytes_total{cluster=\\\"spaceport\\\"}[2m])))\"\ + ,\n \"legendFormat\": \"{{namespace}}/{{pod}}\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 15\n },\n \"unit\": \"Bps\"\n \ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n },\n {\n \"id\": 23,\n \"title\": \"Top Pod Network\ + \ TX (Transmit)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 58\n\ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{cluster=\\\ + \"spaceport\\\"}[2m])))\",\n \"legendFormat\": \"{{namespace}}/{{pod}}\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 15\n },\n \"unit\": \"Bps\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n }\n ]\n}" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-shipyard + namespace: default + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +data: + shipyard-resource-monitor.json: "{\n \"annotations\": {\n \"list\": []\n },\n\ + \ \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 1,\n\ + \ \"id\": null,\n \"links\": [],\n \"liveNow\": false,\n \"refresh\": \"15s\"\ + ,\n \"schemaVersion\": 39,\n \"tags\": [\n \"shipyard\",\n \"resource-monitor\"\ + ,\n \"infrastructure\"\n ],\n \"time\": {\n \"from\": \"now-1h\",\n \ + \ \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n\ + \ \"5s\",\n \"10s\",\n \"15s\",\n \"30s\",\n \"1m\",\n\ + \ \"5m\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"\\u2693\ + \ Shipyard Resource Monitor\",\n \"uid\": \"shipyard-resource-monitor\",\n \"\ + version\": 1,\n \"panels\": [\n {\n \"collapsed\": false,\n \"gridPos\"\ + : {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\":\ + \ 0\n },\n \"id\": 100,\n \"title\": \"\\ud83d\\udda5\\ufe0f Systems\ + \ & Host Performance \\u2014 Shipyard Cluster Fleet\",\n \"type\": \"row\"\ + \n },\n {\n \"id\": 1,\n \"title\": \"Online Nodes (Shipyard)\"\ + ,\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n \ + \ \"w\": 6,\n \"x\": 0,\n \"y\": 1\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"count(node_load1{cluster=\\\ + \"shipyard\\\"})\",\n \"legendFormat\": \"Nodes\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"green\",\n \ + \ \"value\": null\n }\n ]\n },\n \ + \ \"unit\": \"none\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\"\ + : \"center\"\n }\n },\n {\n \"id\": 2,\n \"title\": \"Avg\ + \ Cluster CPU Load (1m)\",\n \"type\": \"stat\",\n \"gridPos\": {\n\ + \ \"h\": 4,\n \"w\": 6,\n \"x\": 6,\n \"y\": 1\n \ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"avg(node_load1{cluster=\\\"shipyard\\\"})\",\n \ + \ \"legendFormat\": \"Load\",\n \"refId\": \"A\"\n }\n \ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"thresholds\"\n },\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"blue\",\n \"value\": null\n\ + \ },\n {\n \"color\": \"orange\",\n \ + \ \"value\": 2.0\n },\n {\n \ + \ \"color\": \"red\",\n \"value\": 4.0\n }\n\ + \ ]\n },\n \"unit\": \"none\"\n }\n },\n\ + \ \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\"\ + : \"area\",\n \"justifyMode\": \"center\"\n }\n },\n {\n \ + \ \"id\": 3,\n \"title\": \"Total Monitored RAM\",\n \"type\": \"\ + stat\",\n \"gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \"\ + x\": 12,\n \"y\": 1\n },\n \"datasource\": \"Prometheus\",\n\ + \ \"targets\": [\n {\n \"expr\": \"sum(node_memory_MemTotal_bytes{cluster=\\\ + \"shipyard\\\"})\",\n \"legendFormat\": \"RAM\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"purple\",\n \ + \ \"value\": null\n }\n ]\n },\n\ + \ \"unit\": \"bytes\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"\ + justifyMode\": \"center\"\n }\n },\n {\n \"id\": 4,\n \"\ + title\": \"Cluster RAM In-Use %\",\n \"type\": \"stat\",\n \"gridPos\"\ + : {\n \"h\": 4,\n \"w\": 6,\n \"x\": 18,\n \"y\":\ + \ 1\n },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"(1 - (sum(node_memory_MemAvailable_bytes{cluster=\\\ + \"shipyard\\\"}) / sum(node_memory_MemTotal_bytes{cluster=\\\"shipyard\\\"})))\ + \ * 100\",\n \"legendFormat\": \"In-Use\",\n \"refId\": \"A\"\ + \n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"thresholds\"\n },\n \ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \"\ + steps\": [\n {\n \"color\": \"green\",\n \ + \ \"value\": null\n },\n {\n \"\ + color\": \"orange\",\n \"value\": 75\n },\n \ + \ {\n \"color\": \"red\",\n \"value\": 90\n\ + \ }\n ]\n },\n \"unit\": \"percent\"\ + \n }\n },\n \"options\": {\n \"colorMode\": \"value\"\ + ,\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\n \ + \ }\n },\n {\n \"id\": 5,\n \"title\": \"CPU Utilization by\ + \ Host (%)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 5\n },\n\ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\\\ + \"idle\\\", cluster=\\\"shipyard\\\"}[2m])) * 100)\",\n \"legendFormat\"\ + : \"{{instance}}\",\n \"refId\": \"A\"\n }\n ],\n \"\ + fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \ + \ \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \ + \ \"drawStyle\": \"line\",\n \"lineInterpolation\": \"smooth\"\ + ,\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n },\n\ + \ \"unit\": \"percent\",\n \"min\": 0,\n \"max\": 100\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 6,\n \ + \ \"title\": \"1-Minute Load Average by Host\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 5\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"node_load1{cluster=\\\"shipyard\\\ + \"}\",\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"\ + A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"none\",\n \ + \ \"min\": 0\n }\n },\n \"options\": {\n \"legend\"\ + : {\n \"displayMode\": \"table\",\n \"placement\": \"bottom\"\ + ,\n \"calcs\": [\n \"mean\",\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n }\n }\n },\n {\n \ + \ \"id\": 7,\n \"title\": \"Memory Used % by Host\",\n \"type\": \"\ + timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \ + \ \"x\": 0,\n \"y\": 13\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"(1 - (node_memory_MemAvailable_bytes{cluster=\\\ + \"shipyard\\\"} / node_memory_MemTotal_bytes{cluster=\\\"shipyard\\\"})) * 100\"\ + ,\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n\ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"\ + lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \"\ + fillOpacity\": 15\n },\n \"unit\": \"percent\",\n \"\ + min\": 0,\n \"max\": 100\n }\n },\n \"options\": {\n\ + \ \"legend\": {\n \"displayMode\": \"table\",\n \"placement\"\ + : \"bottom\",\n \"calcs\": [\n \"mean\",\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n }\n }\n },\n {\n \ + \ \"id\": 8,\n \"title\": \"Available RAM by Host (GiB)\",\n \"type\"\ + : \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n\ + \ \"x\": 12,\n \"y\": 13\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"node_memory_MemAvailable_bytes{cluster=\\\ + \"shipyard\\\"}\",\n \"legendFormat\": \"{{instance}}\",\n \"\ + refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"bytes\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + min\"\n ]\n }\n }\n },\n {\n \"id\": 9,\n \ + \ \"title\": \"Root (/) Filesystem Usage %\",\n \"type\": \"bargauge\",\n\ + \ \"gridPos\": {\n \"h\": 8,\n \"w\": 8,\n \"x\": 0,\n\ + \ \"y\": 21\n },\n \"datasource\": \"Prometheus\",\n \"\ + targets\": [\n {\n \"expr\": \"(1 - (node_filesystem_avail_bytes{mountpoint=\\\ + \"/\\\", cluster=\\\"shipyard\\\"} / node_filesystem_size_bytes{mountpoint=\\\"\ + /\\\", cluster=\\\"shipyard\\\"})) * 100\",\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n\ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\ + \n },\n \"thresholds\": {\n \"mode\": \"absolute\"\ + ,\n \"steps\": [\n {\n \"color\": \"green\"\ + ,\n \"value\": null\n },\n {\n \ + \ \"color\": \"orange\",\n \"value\": 75\n \ + \ },\n {\n \"color\": \"red\",\n \ + \ \"value\": 90\n }\n ]\n },\n \"unit\"\ + : \"percent\",\n \"min\": 0,\n \"max\": 100\n }\n \ + \ },\n \"options\": {\n \"displayMode\": \"gradient\",\n \ + \ \"orientation\": \"horizontal\",\n \"showUnfilled\": true\n }\n\ + \ },\n {\n \"id\": 10,\n \"title\": \"Disk Read Throughput\",\n\ + \ \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \ + \ \"w\": 8,\n \"x\": 8,\n \"y\": 21\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"sum\ + \ by (instance) (rate(node_disk_read_bytes_total{cluster=\\\"shipyard\\\"}[2m]))\"\ + ,\n \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n\ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n\ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"\ + lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \"\ + fillOpacity\": 10\n },\n \"unit\": \"Bps\"\n }\n \ + \ },\n \"options\": {\n \"legend\": {\n \"displayMode\"\ + : \"table\",\n \"placement\": \"bottom\",\n \"calcs\": [\n \ + \ \"mean\",\n \"max\"\n ]\n }\n }\n\ + \ },\n {\n \"id\": 11,\n \"title\": \"Disk Write Throughput\"\ + ,\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n\ + \ \"w\": 8,\n \"x\": 16,\n \"y\": 21\n },\n \"\ + datasource\": \"Prometheus\",\n \"targets\": [\n {\n \"expr\"\ + : \"sum by (instance) (rate(node_disk_written_bytes_total{cluster=\\\"shipyard\\\ + \"}[2m]))\",\n \"legendFormat\": \"{{instance}}\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10\n },\n \"unit\": \"Bps\"\n \ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n },\n {\n \"id\": 12,\n \"title\": \"Host Network Receive\ + \ (RX)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \"h\"\ + : 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 29\n },\n \ + \ \"datasource\": \"Prometheus\",\n \"targets\": [\n {\n \ + \ \"expr\": \"sum by (instance) (rate(node_network_receive_bytes_total{device!~\\\ + \"lo|veth.*|flannel.*|cali.*\\\", cluster=\\\"shipyard\\\"}[2m]))\",\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n }\n\ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"drawStyle\": \"line\",\n \"lineInterpolation\":\ + \ \"smooth\",\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n\ + \ },\n \"unit\": \"Bps\"\n }\n },\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"mean\",\n \ + \ \"max\"\n ]\n }\n }\n },\n {\n \"id\"\ + : 13,\n \"title\": \"Host Network Transmit (TX)\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 12,\n \"y\": 29\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"sum by (instance) (rate(node_network_transmit_bytes_total{device!~\\\ + \"lo|veth.*|flannel.*|cali.*\\\", cluster=\\\"shipyard\\\"}[2m]))\",\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"refId\": \"A\"\n }\n\ + \ ],\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"drawStyle\": \"line\",\n \"lineInterpolation\":\ + \ \"smooth\",\n \"lineWidth\": 2,\n \"fillOpacity\": 15\n\ + \ },\n \"unit\": \"Bps\"\n }\n },\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"mean\",\n \ + \ \"max\"\n ]\n }\n }\n },\n {\n \"collapsed\"\ + : false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \ + \ \"x\": 0,\n \"y\": 37\n },\n \"id\": 200,\n \"title\"\ + : \"\\ud83d\\udce6 Docker Swarm Containers & Service Stacks \\u2014 Shipyard Fleet\"\ + ,\n \"type\": \"row\"\n },\n {\n \"id\": 14,\n \"title\"\ + : \"Monitored Swarm Containers\",\n \"type\": \"stat\",\n \"gridPos\"\ + : {\n \"h\": 4,\n \"w\": 6,\n \"x\": 0,\n \"y\": 38\n\ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"count(container_last_seen{cluster=\\\"shipyard\\\"\ + , name!=\\\"\\\"}) or count(container_last_seen{cluster=\\\"shipyard\\\"})\",\n\ + \ \"legendFormat\": \"Containers\",\n \"refId\": \"A\"\n \ + \ }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n \ + \ \"color\": {\n \"mode\": \"thresholds\"\n },\n \ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\"\ + : [\n {\n \"color\": \"green\",\n \"\ + value\": null\n }\n ]\n },\n \"unit\"\ + : \"none\"\n }\n },\n \"options\": {\n \"colorMode\":\ + \ \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\"\ + \n }\n },\n {\n \"id\": 15,\n \"title\": \"Active Swarm Nodes\"\ + ,\n \"type\": \"stat\",\n \"gridPos\": {\n \"h\": 4,\n \ + \ \"w\": 6,\n \"x\": 6,\n \"y\": 38\n },\n \"datasource\"\ + : \"Prometheus\",\n \"targets\": [\n {\n \"expr\": \"count(up{cluster=\\\ + \"shipyard\\\", job=\\\"node-exporter\\\"})\",\n \"legendFormat\": \"\ + Swarm Nodes\",\n \"refId\": \"A\"\n }\n ],\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"color\": {\n \"mode\": \"\ + thresholds\"\n },\n \"thresholds\": {\n \"mode\"\ + : \"absolute\",\n \"steps\": [\n {\n \"\ + color\": \"blue\",\n \"value\": null\n }\n \ + \ ]\n },\n \"unit\": \"none\"\n }\n },\n \ + \ \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\":\ + \ \"area\",\n \"justifyMode\": \"center\"\n }\n },\n {\n \ + \ \"id\": 16,\n \"title\": \"Total Container CPU Load\",\n \"type\"\ + : \"stat\",\n \"gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \ + \ \"x\": 12,\n \"y\": 38\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"sum(rate(container_cpu_usage_seconds_total{cluster=\\\ + \"shipyard\\\"}[2m]))\",\n \"legendFormat\": \"Cores\",\n \"\ + refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"orange\",\n \ + \ \"value\": null\n }\n ]\n },\n\ + \ \"unit\": \"none\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"\ + justifyMode\": \"center\"\n }\n },\n {\n \"id\": 17,\n \"\ + title\": \"Total Container Working RAM\",\n \"type\": \"stat\",\n \"\ + gridPos\": {\n \"h\": 4,\n \"w\": 6,\n \"x\": 18,\n \ + \ \"y\": 38\n },\n \"datasource\": \"Prometheus\",\n \"targets\"\ + : [\n {\n \"expr\": \"sum(container_memory_working_set_bytes{cluster=\\\ + \"shipyard\\\"})\",\n \"legendFormat\": \"RAM\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n\ + \ \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"purple\",\n \ + \ \"value\": null\n }\n ]\n },\n\ + \ \"unit\": \"bytes\"\n }\n },\n \"options\": {\n \ + \ \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"\ + justifyMode\": \"center\"\n }\n },\n {\n \"id\": 18,\n \"\ + title\": \"Container CPU Usage by Service/Task\",\n \"type\": \"timeseries\"\ + ,\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\"\ + : 0,\n \"y\": 42\n },\n \"datasource\": \"Prometheus\",\n \ + \ \"targets\": [\n {\n \"expr\": \"topk(10, sum by (instance,\ + \ id) (rate(container_cpu_usage_seconds_total{cluster=\\\"shipyard\\\"}[2m])))\"\ + ,\n \"legendFormat\": \"{{instance}} ({{id}})\",\n \"refId\"\ + : \"A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\":\ + \ {\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 15\n },\n \"unit\": \"none\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"lastNotNull\",\n \"\ + max\"\n ]\n }\n }\n },\n {\n \"id\": 19,\n \ + \ \"title\": \"Container Memory Working Set by Service/Task\",\n \"type\"\ + : \"timeseries\",\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n\ + \ \"x\": 12,\n \"y\": 42\n },\n \"datasource\": \"Prometheus\"\ + ,\n \"targets\": [\n {\n \"expr\": \"topk(10, sum by (instance,\ + \ id) (container_memory_working_set_bytes{cluster=\\\"shipyard\\\"}))\",\n \ + \ \"legendFormat\": \"{{instance}} ({{id}})\",\n \"refId\": \"\ + A\"\n }\n ],\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 15\n },\n \"unit\": \"bytes\"\n \ + \ }\n },\n \"options\": {\n \"legend\": {\n \"displayMode\"\ + : \"table\",\n \"placement\": \"bottom\",\n \"calcs\": [\n \ + \ \"mean\",\n \"lastNotNull\",\n \"max\"\n \ + \ ]\n }\n }\n },\n {\n \"id\": 20,\n \"title\"\ + : \"Container Network Receive (RX)\",\n \"type\": \"timeseries\",\n \ + \ \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \ + \ \"y\": 50\n },\n \"datasource\": \"Prometheus\",\n \"targets\"\ + : [\n {\n \"expr\": \"topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster=\\\ + \"shipyard\\\"}[2m])))\",\n \"legendFormat\": \"{{instance}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 15\n },\n \"unit\": \"Bps\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n },\n {\n \"id\": 21,\n \"title\": \"Container Network\ + \ Transmit (TX)\",\n \"type\": \"timeseries\",\n \"gridPos\": {\n \ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 50\n\ + \ },\n \"datasource\": \"Prometheus\",\n \"targets\": [\n \ + \ {\n \"expr\": \"topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster=\\\ + \"shipyard\\\"}[2m])))\",\n \"legendFormat\": \"{{instance}}\",\n \ + \ \"refId\": \"A\"\n }\n ],\n \"fieldConfig\": {\n \ + \ \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineInterpolation\": \"smooth\",\n \"lineWidth\": 2,\n\ + \ \"fillOpacity\": 15\n },\n \"unit\": \"Bps\"\n\ + \ }\n },\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"mean\",\n \"max\"\n ]\n }\n\ + \ }\n }\n ]\n}"