import json def make_panel_stat(id_val, title, expr, grid_pos, unit="none", color="green"): return { "id": id_val, "title": title, "type": "stat", "gridPos": grid_pos, "datasource": "Prometheus", "targets": [{"expr": expr, "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": color, "value": None}]}, "unit": unit } }, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} } def make_panel_timeseries(id_val, title, targets, grid_pos, unit="none"): return { "id": id_val, "title": title, "type": "timeseries", "gridPos": grid_pos, "datasource": "Prometheus", "targets": targets, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15 }, "unit": unit } }, "options": {"tooltip": {"mode": "multi", "sort": "desc"}} } def make_panel_bargauge(id_val, title, expr, grid_pos, unit="percent"): return { "id": id_val, "title": title, "type": "bargauge", "gridPos": grid_pos, "datasource": "Prometheus", "targets": [{"expr": expr, "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": None}, {"color": "yellow", "value": 75}, {"color": "red", "value": 90} ] }, "unit": unit, "min": 0, "max": 100 } }, "options": { "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True } } # ==================== 1. SYSTEMS DASHBOARD ==================== systems_dashboard = { "annotations": {"list": []}, "editable": True, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": None, "links": [], "liveNow": False, "panels": [ { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, "id": 100, "title": "🖥️ Systems Fleet Overview & Capacity", "type": "row" }, make_panel_stat(1, "Online Node Exporters", "count(node_load1)", {"h": 4, "w": 6, "x": 0, "y": 1}, "none", "green"), make_panel_stat(2, "Avg Cluster CPU Load (1m)", "avg(node_load1)", {"h": 4, "w": 6, "x": 6, "y": 1}, "none", "blue"), make_panel_stat(3, "Total Monitored RAM", "sum(node_memory_MemTotal_bytes)", {"h": 4, "w": 6, "x": 12, "y": 1}, "bytes", "purple"), make_panel_stat(4, "Fleet RAM In-Use %", "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", {"h": 4, "w": 6, "x": 18, "y": 1}, "percent", "orange"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, "id": 200, "title": "⚡ CPU Load & Core Utilization", "type": "row" }, make_panel_timeseries(5, "CPU Utilization by Host (%)", [ {"expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 0, "y": 6}, "percent"), make_panel_timeseries(6, "1-Minute Load Average by Host", [ {"expr": "node_load1", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 12, "y": 6}, "none"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}, "id": 300, "title": "🧠 Memory Consumption & Free Capacity", "type": "row" }, make_panel_timeseries(7, "Memory Used % by Host", [ {"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 0, "y": 15}, "percent"), make_panel_timeseries(8, "Available RAM by Host (GiB)", [ {"expr": "node_memory_MemAvailable_bytes", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 12, "y": 15}, "bytes"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23}, "id": 400, "title": "💾 Filesystem & Disk I/O Performance", "type": "row" }, make_panel_bargauge(9, "Root (/) Filesystem Usage %", "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100", {"h": 8, "w": 8, "x": 0, "y": 24}, "percent"), make_panel_timeseries(10, "Disk Read Throughput (Bytes/sec)", [ {"expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))", "legendFormat": "Read {{instance}}", "refId": "A"} ], {"h": 8, "w": 8, "x": 8, "y": 24}, "Bps"), make_panel_timeseries(11, "Disk Write Throughput (Bytes/sec)", [ {"expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))", "legendFormat": "Write {{instance}}", "refId": "A"} ], {"h": 8, "w": 8, "x": 16, "y": 24}, "Bps"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 32}, "id": 500, "title": "🌐 Network Throughput (RX / TX)", "type": "row" }, make_panel_timeseries(12, "Host Network Receive (RX)", [ {"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", "legendFormat": "RX {{instance}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 0, "y": 33}, "Bps"), make_panel_timeseries(13, "Host Network Transmit (TX)", [ {"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", "legendFormat": "TX {{instance}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 12, "y": 33}, "Bps") ], "refresh": "10s", "schemaVersion": 39, "tags": ["infrastructure", "systems", "snarfnet", "node-exporter"], "templating": {"list": []}, "time": {"from": "now-1h", "to": "now"}, "timepicker": {}, "timezone": "browser", "title": "🖥️ Systems & Cluster Host Performance", "uid": "snarfnet-systems" } with open("homelab-gitops-repo/manifests/aerodrome/grafana/systems-dashboard.json", "w", encoding="utf-8") as f: json.dump(systems_dashboard, f, indent=2) print("Created systems-dashboard.json successfully!") # ==================== 2. CONTAINER METRICS DASHBOARD ==================== container_dashboard = { "annotations": {"list": []}, "editable": True, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": None, "links": [], "liveNow": False, "panels": [ { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, "id": 100, "title": "📦 Container Fleet & Pod Metrics", "type": "row" }, make_panel_stat(1, "Monitored Containers", "count(container_last_seen{container!=''})", {"h": 4, "w": 6, "x": 0, "y": 1}, "none", "green"), make_panel_stat(2, "Active Kubernetes Pods", "count(count by (pod, namespace) (container_last_seen{container!=''}))", {"h": 4, "w": 6, "x": 6, "y": 1}, "none", "blue"), make_panel_stat(3, "Total Container CPU Load", "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))", {"h": 4, "w": 6, "x": 12, "y": 1}, "none", "purple"), make_panel_stat(4, "Total Container RAM", "sum(container_memory_working_set_bytes{container!=''})", {"h": 4, "w": 6, "x": 18, "y": 1}, "bytes", "orange"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, "id": 200, "title": "⚡ Top Container CPU Utilization", "type": "row" }, make_panel_timeseries(5, "Top 10 CPU Consuming Pods (Cores)", [ {"expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))", "legendFormat": "{{namespace}} / {{pod}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 0, "y": 6}, "none"), make_panel_timeseries(6, "CPU Throttling by Namespace", [ {"expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))", "legendFormat": "{{namespace}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 12, "y": 6}, "none"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}, "id": 300, "title": "🧠 Container Memory Working Set", "type": "row" }, make_panel_timeseries(7, "Top 10 Memory Consuming Pods", [ {"expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))", "legendFormat": "{{namespace}} / {{pod}}", "refId": "A"} ], {"h": 8, "w": 14, "x": 0, "y": 15}, "bytes"), make_panel_bargauge(8, "Memory by Namespace", "sum by (namespace) (container_memory_working_set_bytes{container!=''})", {"h": 8, "w": 10, "x": 14, "y": 15}, "bytes"), { "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23}, "id": 400, "title": "🌐 Container Network Traffic", "type": "row" }, make_panel_timeseries(9, "Top Pod Network RX (Receive)", [ {"expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))", "legendFormat": "RX {{namespace}}/{{pod}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 0, "y": 24}, "Bps"), make_panel_timeseries(10, "Top Pod Network TX (Transmit)", [ {"expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))", "legendFormat": "TX {{namespace}}/{{pod}}", "refId": "A"} ], {"h": 8, "w": 12, "x": 12, "y": 24}, "Bps") ], "refresh": "10s", "schemaVersion": 39, "tags": ["kubernetes", "containers", "cadvisor", "snarfnet", "workloads"], "templating": {"list": []}, "time": {"from": "now-1h", "to": "now"}, "timepicker": {}, "timezone": "browser", "title": "📦 Kubernetes Containers & Workloads", "uid": "snarfnet-containers" } with open("homelab-gitops-repo/manifests/aerodrome/grafana/container-metrics-dashboard.json", "w", encoding="utf-8") as f: json.dump(container_dashboard, f, indent=2) print("Created container-metrics-dashboard.json successfully!")