import json import requests import copy import urllib3 urllib3.disable_warnings() auth = ("admin", "Katzir476!!!") base_url = "https://grafana.snarfnet.net" # Load templates with open("homelab-gitops-repo/dashboards/grafana/systems-dashboard.json") as f: sys_template = json.load(f) with open("homelab-gitops-repo/dashboards/grafana/container-metrics-dashboard.json") as f: cnt_template = json.load(f) def build_cluster_dashboard(cluster_key, cluster_title, folder_id, folder_uid, cluster_emoji): dash = { "annotations": {"list": []}, "editable": True, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": None, "links": [], "liveNow": False, "refresh": "15s", "schemaVersion": 39, "tags": [cluster_key, "resource-monitor", "infrastructure"], "time": {"from": "now-1h", "to": "now"}, "timepicker": {"refresh_intervals": ["5s", "10s", "15s", "30s", "1m", "5m"]}, "timezone": "browser", "title": f"{cluster_emoji} {cluster_title} Resource Monitor", "uid": f"{cluster_key}-resource-monitor", "version": 1, "panels": [] } panels = [] current_y = 0 # ========================================== # GROUP 1: Systems & Cluster Host Performance # ========================================== # Elevate Systems Row Header panels.append({ "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y}, "id": 100, "title": f"🖥️ Systems & Host Performance — {cluster_title} Cluster Fleet", "type": "row" }) current_y += 1 # Systems Stat Panels (height 4) stat_panels = [ { "id": 1, "title": f"Online Nodes ({cluster_title})", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'count(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Nodes", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 2, "title": "Avg Cluster CPU Load (1m)", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'avg(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Load", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}, {"color": "orange", "value": 2.0}, {"color": "red", "value": 4.0}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 3, "title": "Total Monitored RAM", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})', "legendFormat": "RAM", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 4, "title": "Cluster RAM In-Use %", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'(1 - (sum(node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}) / sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}}))) * 100', "legendFormat": "In-Use", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} } ] panels.extend(stat_panels) current_y += 4 # Systems CPU Timeseries panels.append({ "id": 5, "title": "CPU Utilization by Host (%)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'100 - (avg by (instance) (rate(node_cpu_seconds_total{{mode="idle", cluster="{cluster_key}"}}[2m])) * 100)', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) panels.append({ "id": 6, "title": "1-Minute Load Average by Host", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'node_load1{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none", "min": 0}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) current_y += 8 # Systems Memory Timeseries panels.append({ "id": 7, "title": "Memory Used % by Host", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'(1 - (node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}} / node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) panels.append({ "id": 8, "title": "Available RAM by Host (GiB)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "bytes"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "min"]}} }) current_y += 8 # Filesystem & Disk I/O panels.append({ "id": 9, "title": "Root (/) Filesystem Usage %", "type": "bargauge", "gridPos": {"h": 8, "w": 8, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'(1 - (node_filesystem_avail_bytes{{mountpoint="/", cluster="{cluster_key}"}} / node_filesystem_size_bytes{{mountpoint="/", cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent", "min": 0, "max": 100}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True} }) panels.append({ "id": 10, "title": "Disk Read Throughput", "type": "timeseries", "gridPos": {"h": 8, "w": 8, "x": 8, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum by (instance) (rate(node_disk_read_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) panels.append({ "id": 11, "title": "Disk Write Throughput", "type": "timeseries", "gridPos": {"h": 8, "w": 8, "x": 16, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum by (instance) (rate(node_disk_written_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) current_y += 8 # Network RX/TX panels.append({ "id": 12, "title": "Host Network Receive (RX)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum by (instance) (rate(node_network_receive_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) panels.append({ "id": 13, "title": "Host Network Transmit (TX)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum by (instance) (rate(node_network_transmit_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) current_y += 8 # ========================================== # GROUP 2: Containers & Workloads # ========================================== if cluster_key in ["aerodrome", "spaceport"]: cadvisor_filter = f'cluster="{cluster_key}"' if cluster_key == "spaceport" else 'instance=~"sninfr2.*"' panels.append({ "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y}, "id": 200, "title": f"📦 Kubernetes Containers & Workloads — {cluster_title} Cluster", "type": "row" }) current_y += 1 # Container Stats panels.extend([ { "id": 14, "title": "Monitored Containers", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'count(container_last_seen{{container!="", {cadvisor_filter}}})', "legendFormat": "Containers", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 15, "title": "Active Kubernetes Pods", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'count(count by (pod, namespace) (container_last_seen{{container!="", {cadvisor_filter}}}))', "legendFormat": "Pods", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 16, "title": "Total Container CPU Load", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum(rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m]))', "legendFormat": "Cores", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 17, "title": "Total Container Working RAM", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum(container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "RAM", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} } ]) current_y += 4 # Container CPU panels.append({ "id": 18, "title": "Top 10 CPU Consuming Pods (Cores)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) panels.append({ "id": 19, "title": "CPU Throttling by Namespace", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{{{cadvisor_filter}}}[2m]))', "legendFormat": "{{namespace}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) current_y += 8 # Container Memory panels.append({ "id": 20, "title": "Top 10 Memory Consuming Pods", "type": "timeseries", "gridPos": {"h": 8, "w": 14, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}}))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) panels.append({ "id": 21, "title": "Memory Working Set by Namespace", "type": "bargauge", "gridPos": {"h": 8, "w": 10, "x": 14, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'sum by (namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "{{namespace}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True} }) current_y += 8 # Container Network panels.append({ "id": 22, "title": "Top Pod Network RX (Receive)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) panels.append({ "id": 23, "title": "Top Pod Network TX (Transmit)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) current_y += 8 else: # Shipyard (Docker Swarm) panels.append({ "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y}, "id": 200, "title": "📦 Docker Swarm Containers & Service Stacks — Shipyard Fleet", "type": "row" }) current_y += 1 panels.extend([ { "id": 14, "title": "Monitored Swarm Containers", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'count(container_last_seen{cluster="shipyard", name!=""}) or count(container_last_seen{cluster="shipyard"})', "legendFormat": "Containers", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 15, "title": "Active Swarm Nodes", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'count(up{cluster="shipyard", job="node-exporter"})', "legendFormat": "Swarm Nodes", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 16, "title": "Total Container CPU Load", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'sum(rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m]))', "legendFormat": "Cores", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} }, { "id": 17, "title": "Total Container Working RAM", "type": "stat", "gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'sum(container_memory_working_set_bytes{cluster="shipyard"})', "legendFormat": "RAM", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}}, "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} } ]) current_y += 4 # Shipyard CPU & Memory panels.append({ "id": 18, "title": "Container CPU Usage by Service/Task", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'topk(10, sum by (instance, id) (rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) panels.append({ "id": 19, "title": "Container Memory Working Set by Service/Task", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'topk(10, sum by (instance, id) (container_memory_working_set_bytes{cluster="shipyard"}))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}} }) current_y += 8 # Shipyard Network RX/TX panels.append({ "id": 20, "title": "Container Network Receive (RX)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) panels.append({ "id": 21, "title": "Container Network Transmit (TX)", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y}, "datasource": "Prometheus", "targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}], "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}} }) current_y += 8 dash["panels"] = panels return dash clusters = [ {"key": "aerodrome", "title": "Aerodrome", "folder_id": 5, "folder_uid": "folder-aerodrome", "emoji": "✈️"}, {"key": "spaceport", "title": "Spaceport", "folder_id": 6, "folder_uid": "folder-spaceport", "emoji": "🚀"}, {"key": "shipyard", "title": "Shipyard", "folder_id": 7, "folder_uid": "folder-shipyard", "emoji": "⚓"} ] # 1. Generate and Upload Dashboards for c in clusters: dash_json = build_cluster_dashboard(c["key"], c["title"], c["folder_id"], c["folder_uid"], c["emoji"]) # Save to local gitops repo filename = f"homelab-gitops-repo/dashboards/grafana/{c['key']}-resource-monitor.json" with open(filename, "w", encoding="utf-8") as f: json.dump(dash_json, f, indent=2) print(f"Saved {filename}") # Upload to Grafana API payload = { "dashboard": dash_json, "folderId": c["folder_id"], "folderUid": c["folder_uid"], "overwrite": True } resp = requests.post(f"{base_url}/api/dashboards/db", json=payload, auth=auth, verify=False, timeout=10) print(f"Uploaded {c['title']} dashboard to Grafana: {resp.status_code} - {resp.json().get('status')}") # 2. Delete old dashboards from AI Stack folder for old_uid in ["snarfnet-systems", "snarfnet-containers"]: del_resp = requests.delete(f"{base_url}/api/dashboards/uid/{old_uid}", auth=auth, verify=False, timeout=10) print(f"Deleted old dashboard {old_uid}: {del_resp.status_code}") print("\nDashboard generation and deployment complete!")