diff --git a/manifests/aerodrome/grafana/container-metrics-dashboard.json b/manifests/aerodrome/grafana/container-metrics-dashboard.json new file mode 100644 index 0000000..8820bff --- /dev/null +++ b/manifests/aerodrome/grafana/container-metrics-dashboard.json @@ -0,0 +1,476 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udce6 Container Fleet & Pod Metrics", + "type": "row" + }, + { + "id": 1, + "title": "Monitored Containers", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(container_last_seen{container!=''})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Active Kubernetes Pods", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(count by (pod, namespace) (container_last_seen{container!=''}))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Container CPU Load", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Total Container RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=''})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "\u26a1 Top Container CPU Utilization", + "type": "row" + }, + { + "id": 5, + "title": "Top 10 CPU Consuming Pods (Cores)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))", + "legendFormat": "{{namespace}} / {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "CPU Throttling by Namespace", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 300, + "title": "\ud83e\udde0 Container Memory Working Set", + "type": "row" + }, + { + "id": 7, + "title": "Top 10 Memory Consuming Pods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))", + "legendFormat": "{{namespace}} / {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Memory by Namespace", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=''})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "bytes", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 400, + "title": "\ud83c\udf10 Container Network Traffic", + "type": "row" + }, + { + "id": 9, + "title": "Top Pod Network RX (Receive)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))", + "legendFormat": "RX {{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "title": "Top Pod Network TX (Transmit)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))", + "legendFormat": "TX {{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "kubernetes", + "containers", + "cadvisor", + "snarfnet", + "workloads" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "\ud83d\udce6 Kubernetes Containers & Workloads", + "uid": "snarfnet-containers" +} \ No newline at end of file diff --git a/manifests/aerodrome/grafana/grafana.yaml b/manifests/aerodrome/grafana/grafana.yaml index d2fab0e..d3af6e2 100644 --- a/manifests/aerodrome/grafana/grafana.yaml +++ b/manifests/aerodrome/grafana/grafana.yaml @@ -120,6 +120,12 @@ spec: - mountPath: /var/lib/grafana-dashboards/ai-stack-dashboard.json name: ai-stack-dashboard subPath: ai-stack-dashboard.json + - mountPath: /var/lib/grafana-dashboards/systems-dashboard.json + name: systems-dashboard + subPath: systems-dashboard.json + - mountPath: /var/lib/grafana-dashboards/container-metrics-dashboard.json + name: container-metrics-dashboard + subPath: container-metrics-dashboard.json dnsPolicy: ClusterFirst enableServiceLinks: true initContainers: @@ -175,6 +181,12 @@ spec: defaultMode: 420 name: grafana-dashboard-ai-stack name: ai-stack-dashboard + - configMap: + name: grafana-dashboard-systems + name: systems-dashboard + - configMap: + name: grafana-dashboard-containers + name: container-metrics-dashboard --- apiVersion: v1 kind: Service @@ -843,3 +855,1096 @@ data: "uid": "snarfnet-ai-stack", "version": 1 } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-systems + namespace: default + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +data: + systems-dashboard.json: |- + { + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udda5\ufe0f Systems Fleet Overview & Capacity", + "type": "row" + }, + { + "id": 1, + "title": "Online Node Exporters", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(node_load1)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Avg Cluster CPU Load (1m)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "avg(node_load1)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Monitored RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(node_memory_MemTotal_bytes)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Fleet RAM In-Use %", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "\u26a1 CPU Load & Core Utilization", + "type": "row" + }, + { + "id": 5, + "title": "CPU Utilization by Host (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "1-Minute Load Average by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_load1", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 300, + "title": "\ud83e\udde0 Memory Consumption & Free Capacity", + "type": "row" + }, + { + "id": 7, + "title": "Memory Used % by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Available RAM by Host (GiB)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 400, + "title": "\ud83d\udcbe Filesystem & Disk I/O Performance", + "type": "row" + }, + { + "id": 9, + "title": "Root (/) Filesystem Usage %", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 10, + "title": "Disk Read Throughput (Bytes/sec)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))", + "legendFormat": "Read {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "title": "Disk Write Throughput (Bytes/sec)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))", + "legendFormat": "Write {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 500, + "title": "\ud83c\udf10 Network Throughput (RX / TX)", + "type": "row" + }, + { + "id": 12, + "title": "Host Network Receive (RX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", + "legendFormat": "RX {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "title": "Host Network Transmit (TX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", + "legendFormat": "TX {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "infrastructure", + "systems", + "snarfnet", + "node-exporter" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "\ud83d\udda5\ufe0f Systems & Cluster Host Performance", + "uid": "snarfnet-systems" + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-containers + namespace: default + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +data: + container-metrics-dashboard.json: |- + { + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udce6 Container Fleet & Pod Metrics", + "type": "row" + }, + { + "id": 1, + "title": "Monitored Containers", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(container_last_seen{container!=''})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Active Kubernetes Pods", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(count by (pod, namespace) (container_last_seen{container!=''}))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Container CPU Load", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Total Container RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=''})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "\u26a1 Top Container CPU Utilization", + "type": "row" + }, + { + "id": 5, + "title": "Top 10 CPU Consuming Pods (Cores)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))", + "legendFormat": "{{namespace}} / {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "CPU Throttling by Namespace", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 300, + "title": "\ud83e\udde0 Container Memory Working Set", + "type": "row" + }, + { + "id": 7, + "title": "Top 10 Memory Consuming Pods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))", + "legendFormat": "{{namespace}} / {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Memory by Namespace", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=''})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "bytes", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 400, + "title": "\ud83c\udf10 Container Network Traffic", + "type": "row" + }, + { + "id": 9, + "title": "Top Pod Network RX (Receive)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))", + "legendFormat": "RX {{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "title": "Top Pod Network TX (Transmit)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))", + "legendFormat": "TX {{namespace}}/{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "kubernetes", + "containers", + "cadvisor", + "snarfnet", + "workloads" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "\ud83d\udce6 Kubernetes Containers & Workloads", + "uid": "snarfnet-containers" + } diff --git a/manifests/aerodrome/grafana/systems-dashboard.json b/manifests/aerodrome/grafana/systems-dashboard.json new file mode 100644 index 0000000..a9ab704 --- /dev/null +++ b/manifests/aerodrome/grafana/systems-dashboard.json @@ -0,0 +1,595 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\ud83d\udda5\ufe0f Systems Fleet Overview & Capacity", + "type": "row" + }, + { + "id": 1, + "title": "Online Node Exporters", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(node_load1)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Avg Cluster CPU Load (1m)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "avg(node_load1)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Total Monitored RAM", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(node_memory_MemTotal_bytes)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Fleet RAM In-Use %", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "\u26a1 CPU Load & Core Utilization", + "type": "row" + }, + { + "id": 5, + "title": "CPU Utilization by Host (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "1-Minute Load Average by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_load1", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "none" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 300, + "title": "\ud83e\udde0 Memory Consumption & Free Capacity", + "type": "row" + }, + { + "id": 7, + "title": "Memory Used % by Host", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "percent" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Available RAM by Host (GiB)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes", + "legendFormat": "{{cluster}} - {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "bytes" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 400, + "title": "\ud83d\udcbe Filesystem & Disk I/O Performance", + "type": "row" + }, + { + "id": 9, + "title": "Root (/) Filesystem Usage %", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + } + }, + { + "id": 10, + "title": "Disk Read Throughput (Bytes/sec)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))", + "legendFormat": "Read {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "title": "Disk Write Throughput (Bytes/sec)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))", + "legendFormat": "Write {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 500, + "title": "\ud83c\udf10 Network Throughput (RX / TX)", + "type": "row" + }, + { + "id": 12, + "title": "Host Network Receive (RX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", + "legendFormat": "RX {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "title": "Host Network Transmit (TX)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", + "legendFormat": "TX {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": "Bps" + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "infrastructure", + "systems", + "snarfnet", + "node-exporter" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "\ud83d\udda5\ufe0f Systems & Cluster Host Performance", + "uid": "snarfnet-systems" +} \ No newline at end of file