Files
SnarfCode/scripts/build_and_upload_dashboards.py

468 lines
26 KiB
Python

import json
import requests
import copy
import urllib3
urllib3.disable_warnings()
auth = ("admin", "Katzir476!!!")
base_url = "https://grafana.snarfnet.net"
# Load templates
with open("homelab-gitops-repo/dashboards/grafana/systems-dashboard.json") as f:
sys_template = json.load(f)
with open("homelab-gitops-repo/dashboards/grafana/container-metrics-dashboard.json") as f:
cnt_template = json.load(f)
def build_cluster_dashboard(cluster_key, cluster_title, folder_id, folder_uid, cluster_emoji):
dash = {
"annotations": {"list": []},
"editable": True,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": None,
"links": [],
"liveNow": False,
"refresh": "15s",
"schemaVersion": 39,
"tags": [cluster_key, "resource-monitor", "infrastructure"],
"time": {"from": "now-1h", "to": "now"},
"timepicker": {"refresh_intervals": ["5s", "10s", "15s", "30s", "1m", "5m"]},
"timezone": "browser",
"title": f"{cluster_emoji} {cluster_title} Resource Monitor",
"uid": f"{cluster_key}-resource-monitor",
"version": 1,
"panels": []
}
panels = []
current_y = 0
# ==========================================
# GROUP 1: Systems & Cluster Host Performance
# ==========================================
# Elevate Systems Row Header
panels.append({
"collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
"id": 100,
"title": f"🖥️ Systems & Host Performance — {cluster_title} Cluster Fleet",
"type": "row"
})
current_y += 1
# Systems Stat Panels (height 4)
stat_panels = [
{
"id": 1,
"title": f"Online Nodes ({cluster_title})",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'count(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Nodes", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 2,
"title": "Avg Cluster CPU Load (1m)",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'avg(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Load", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}, {"color": "orange", "value": 2.0}, {"color": "red", "value": 4.0}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 3,
"title": "Total Monitored RAM",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})', "legendFormat": "RAM", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 4,
"title": "Cluster RAM In-Use %",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'(1 - (sum(node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}) / sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}}))) * 100', "legendFormat": "In-Use", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
}
]
panels.extend(stat_panels)
current_y += 4
# Systems CPU Timeseries
panels.append({
"id": 5,
"title": "CPU Utilization by Host (%)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'100 - (avg by (instance) (rate(node_cpu_seconds_total{{mode="idle", cluster="{cluster_key}"}}[2m])) * 100)', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 6,
"title": "1-Minute Load Average by Host",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'node_load1{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none", "min": 0}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
current_y += 8
# Systems Memory Timeseries
panels.append({
"id": 7,
"title": "Memory Used % by Host",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'(1 - (node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}} / node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 8,
"title": "Available RAM by Host (GiB)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "bytes"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "min"]}}
})
current_y += 8
# Filesystem & Disk I/O
panels.append({
"id": 9,
"title": "Root (/) Filesystem Usage %",
"type": "bargauge",
"gridPos": {"h": 8, "w": 8, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'(1 - (node_filesystem_avail_bytes{{mountpoint="/", cluster="{cluster_key}"}} / node_filesystem_size_bytes{{mountpoint="/", cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent", "min": 0, "max": 100}},
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
})
panels.append({
"id": 10,
"title": "Disk Read Throughput",
"type": "timeseries",
"gridPos": {"h": 8, "w": 8, "x": 8, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_disk_read_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 11,
"title": "Disk Write Throughput",
"type": "timeseries",
"gridPos": {"h": 8, "w": 8, "x": 16, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_disk_written_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
# Network RX/TX
panels.append({
"id": 12,
"title": "Host Network Receive (RX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_network_receive_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 13,
"title": "Host Network Transmit (TX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_network_transmit_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
# ==========================================
# GROUP 2: Containers & Workloads
# ==========================================
if cluster_key in ["aerodrome", "spaceport"]:
cadvisor_filter = f'cluster="{cluster_key}"' if cluster_key == "spaceport" else 'instance=~"sninfr2.*"'
panels.append({
"collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
"id": 200,
"title": f"📦 Kubernetes Containers & Workloads — {cluster_title} Cluster",
"type": "row"
})
current_y += 1
# Container Stats
panels.extend([
{
"id": 14,
"title": "Monitored Containers",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'count(container_last_seen{{container!="", {cadvisor_filter}}})', "legendFormat": "Containers", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 15,
"title": "Active Kubernetes Pods",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'count(count by (pod, namespace) (container_last_seen{{container!="", {cadvisor_filter}}}))', "legendFormat": "Pods", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 16,
"title": "Total Container CPU Load",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum(rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m]))', "legendFormat": "Cores", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 17,
"title": "Total Container Working RAM",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum(container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "RAM", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
}
])
current_y += 4
# Container CPU
panels.append({
"id": 18,
"title": "Top 10 CPU Consuming Pods (Cores)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 19,
"title": "CPU Throttling by Namespace",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{{{cadvisor_filter}}}[2m]))', "legendFormat": "{{namespace}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
current_y += 8
# Container Memory
panels.append({
"id": 20,
"title": "Top 10 Memory Consuming Pods",
"type": "timeseries",
"gridPos": {"h": 8, "w": 14, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}}))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 21,
"title": "Memory Working Set by Namespace",
"type": "bargauge",
"gridPos": {"h": 8, "w": 10, "x": 14, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "{{namespace}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}},
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
})
current_y += 8
# Container Network
panels.append({
"id": 22,
"title": "Top Pod Network RX (Receive)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 23,
"title": "Top Pod Network TX (Transmit)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
else:
# Shipyard (Docker Swarm)
panels.append({
"collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
"id": 200,
"title": "📦 Docker Swarm Containers & Service Stacks — Shipyard Fleet",
"type": "row"
})
current_y += 1
panels.extend([
{
"id": 14,
"title": "Monitored Swarm Containers",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'count(container_last_seen{cluster="shipyard", name!=""}) or count(container_last_seen{cluster="shipyard"})', "legendFormat": "Containers", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 15,
"title": "Active Swarm Nodes",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'count(up{cluster="shipyard", job="node-exporter"})', "legendFormat": "Swarm Nodes", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 16,
"title": "Total Container CPU Load",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'sum(rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m]))', "legendFormat": "Cores", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 17,
"title": "Total Container Working RAM",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'sum(container_memory_working_set_bytes{cluster="shipyard"})', "legendFormat": "RAM", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
}
])
current_y += 4
# Shipyard CPU & Memory
panels.append({
"id": 18,
"title": "Container CPU Usage by Service/Task",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance, id) (rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 19,
"title": "Container Memory Working Set by Service/Task",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance, id) (container_memory_working_set_bytes{cluster="shipyard"}))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
current_y += 8
# Shipyard Network RX/TX
panels.append({
"id": 20,
"title": "Container Network Receive (RX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 21,
"title": "Container Network Transmit (TX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
dash["panels"] = panels
return dash
clusters = [
{"key": "aerodrome", "title": "Aerodrome", "folder_id": 5, "folder_uid": "folder-aerodrome", "emoji": "✈️"},
{"key": "spaceport", "title": "Spaceport", "folder_id": 6, "folder_uid": "folder-spaceport", "emoji": "🚀"},
{"key": "shipyard", "title": "Shipyard", "folder_id": 7, "folder_uid": "folder-shipyard", "emoji": "⚓"}
]
# 1. Generate and Upload Dashboards
for c in clusters:
dash_json = build_cluster_dashboard(c["key"], c["title"], c["folder_id"], c["folder_uid"], c["emoji"])
# Save to local gitops repo
filename = f"homelab-gitops-repo/dashboards/grafana/{c['key']}-resource-monitor.json"
with open(filename, "w", encoding="utf-8") as f:
json.dump(dash_json, f, indent=2)
print(f"Saved {filename}")
# Upload to Grafana API
payload = {
"dashboard": dash_json,
"folderId": c["folder_id"],
"folderUid": c["folder_uid"],
"overwrite": True
}
resp = requests.post(f"{base_url}/api/dashboards/db", json=payload, auth=auth, verify=False, timeout=10)
print(f"Uploaded {c['title']} dashboard to Grafana: {resp.status_code} - {resp.json().get('status')}")
# 2. Delete old dashboards from AI Stack folder
for old_uid in ["snarfnet-systems", "snarfnet-containers"]:
del_resp = requests.delete(f"{base_url}/api/dashboards/uid/{old_uid}", auth=auth, verify=False, timeout=10)
print(f"Deleted old dashboard {old_uid}: {del_resp.status_code}")
print("\nDashboard generation and deployment complete!")