feat: add various infrastructure check and automation scripts
This commit is contained in:
467
scripts/build_and_upload_dashboards.py
Normal file
467
scripts/build_and_upload_dashboards.py
Normal file
@@ -0,0 +1,467 @@
|
||||
import json
|
||||
import requests
|
||||
import copy
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
base_url = "https://grafana.snarfnet.net"
|
||||
|
||||
# Load templates
|
||||
with open("homelab-gitops-repo/dashboards/grafana/systems-dashboard.json") as f:
|
||||
sys_template = json.load(f)
|
||||
|
||||
with open("homelab-gitops-repo/dashboards/grafana/container-metrics-dashboard.json") as f:
|
||||
cnt_template = json.load(f)
|
||||
|
||||
def build_cluster_dashboard(cluster_key, cluster_title, folder_id, folder_uid, cluster_emoji):
|
||||
dash = {
|
||||
"annotations": {"list": []},
|
||||
"editable": True,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": None,
|
||||
"links": [],
|
||||
"liveNow": False,
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [cluster_key, "resource-monitor", "infrastructure"],
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"timepicker": {"refresh_intervals": ["5s", "10s", "15s", "30s", "1m", "5m"]},
|
||||
"timezone": "browser",
|
||||
"title": f"{cluster_emoji} {cluster_title} Resource Monitor",
|
||||
"uid": f"{cluster_key}-resource-monitor",
|
||||
"version": 1,
|
||||
"panels": []
|
||||
}
|
||||
|
||||
panels = []
|
||||
current_y = 0
|
||||
|
||||
# ==========================================
|
||||
# GROUP 1: Systems & Cluster Host Performance
|
||||
# ==========================================
|
||||
# Elevate Systems Row Header
|
||||
panels.append({
|
||||
"collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
|
||||
"id": 100,
|
||||
"title": f"🖥️ Systems & Host Performance — {cluster_title} Cluster Fleet",
|
||||
"type": "row"
|
||||
})
|
||||
current_y += 1
|
||||
|
||||
# Systems Stat Panels (height 4)
|
||||
stat_panels = [
|
||||
{
|
||||
"id": 1,
|
||||
"title": f"Online Nodes ({cluster_title})",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'count(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Nodes", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Avg Cluster CPU Load (1m)",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'avg(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Load", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}, {"color": "orange", "value": 2.0}, {"color": "red", "value": 4.0}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Total Monitored RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})', "legendFormat": "RAM", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Cluster RAM In-Use %",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'(1 - (sum(node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}) / sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}}))) * 100', "legendFormat": "In-Use", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
}
|
||||
]
|
||||
panels.extend(stat_panels)
|
||||
current_y += 4
|
||||
|
||||
# Systems CPU Timeseries
|
||||
panels.append({
|
||||
"id": 5,
|
||||
"title": "CPU Utilization by Host (%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'100 - (avg by (instance) (rate(node_cpu_seconds_total{{mode="idle", cluster="{cluster_key}"}}[2m])) * 100)', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 6,
|
||||
"title": "1-Minute Load Average by Host",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'node_load1{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none", "min": 0}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Systems Memory Timeseries
|
||||
panels.append({
|
||||
"id": 7,
|
||||
"title": "Memory Used % by Host",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'(1 - (node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}} / node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 8,
|
||||
"title": "Available RAM by Host (GiB)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "bytes"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "min"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Filesystem & Disk I/O
|
||||
panels.append({
|
||||
"id": 9,
|
||||
"title": "Root (/) Filesystem Usage %",
|
||||
"type": "bargauge",
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'(1 - (node_filesystem_avail_bytes{{mountpoint="/", cluster="{cluster_key}"}} / node_filesystem_size_bytes{{mountpoint="/", cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent", "min": 0, "max": 100}},
|
||||
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
|
||||
})
|
||||
panels.append({
|
||||
"id": 10,
|
||||
"title": "Disk Read Throughput",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_disk_read_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 11,
|
||||
"title": "Disk Write Throughput",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_disk_written_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Network RX/TX
|
||||
panels.append({
|
||||
"id": 12,
|
||||
"title": "Host Network Receive (RX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_network_receive_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 13,
|
||||
"title": "Host Network Transmit (TX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_network_transmit_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# ==========================================
|
||||
# GROUP 2: Containers & Workloads
|
||||
# ==========================================
|
||||
if cluster_key in ["aerodrome", "spaceport"]:
|
||||
cadvisor_filter = f'cluster="{cluster_key}"' if cluster_key == "spaceport" else 'instance=~"sninfr2.*"'
|
||||
|
||||
panels.append({
|
||||
"collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
|
||||
"id": 200,
|
||||
"title": f"📦 Kubernetes Containers & Workloads — {cluster_title} Cluster",
|
||||
"type": "row"
|
||||
})
|
||||
current_y += 1
|
||||
|
||||
# Container Stats
|
||||
panels.extend([
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Monitored Containers",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'count(container_last_seen{{container!="", {cadvisor_filter}}})', "legendFormat": "Containers", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Active Kubernetes Pods",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'count(count by (pod, namespace) (container_last_seen{{container!="", {cadvisor_filter}}}))', "legendFormat": "Pods", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Total Container CPU Load",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum(rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m]))', "legendFormat": "Cores", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Total Container Working RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum(container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "RAM", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
}
|
||||
])
|
||||
current_y += 4
|
||||
|
||||
# Container CPU
|
||||
panels.append({
|
||||
"id": 18,
|
||||
"title": "Top 10 CPU Consuming Pods (Cores)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 19,
|
||||
"title": "CPU Throttling by Namespace",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{{{cadvisor_filter}}}[2m]))', "legendFormat": "{{namespace}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Container Memory
|
||||
panels.append({
|
||||
"id": 20,
|
||||
"title": "Top 10 Memory Consuming Pods",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 14, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}}))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 21,
|
||||
"title": "Memory Working Set by Namespace",
|
||||
"type": "bargauge",
|
||||
"gridPos": {"h": 8, "w": 10, "x": 14, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "{{namespace}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}},
|
||||
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Container Network
|
||||
panels.append({
|
||||
"id": 22,
|
||||
"title": "Top Pod Network RX (Receive)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 23,
|
||||
"title": "Top Pod Network TX (Transmit)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
else:
|
||||
# Shipyard (Docker Swarm)
|
||||
panels.append({
|
||||
"collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
|
||||
"id": 200,
|
||||
"title": "📦 Docker Swarm Containers & Service Stacks — Shipyard Fleet",
|
||||
"type": "row"
|
||||
})
|
||||
current_y += 1
|
||||
|
||||
panels.extend([
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Monitored Swarm Containers",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'count(container_last_seen{cluster="shipyard", name!=""}) or count(container_last_seen{cluster="shipyard"})', "legendFormat": "Containers", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Active Swarm Nodes",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'count(up{cluster="shipyard", job="node-exporter"})', "legendFormat": "Swarm Nodes", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Total Container CPU Load",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'sum(rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m]))', "legendFormat": "Cores", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Total Container Working RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'sum(container_memory_working_set_bytes{cluster="shipyard"})', "legendFormat": "RAM", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
}
|
||||
])
|
||||
current_y += 4
|
||||
|
||||
# Shipyard CPU & Memory
|
||||
panels.append({
|
||||
"id": 18,
|
||||
"title": "Container CPU Usage by Service/Task",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance, id) (rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 19,
|
||||
"title": "Container Memory Working Set by Service/Task",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance, id) (container_memory_working_set_bytes{cluster="shipyard"}))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Shipyard Network RX/TX
|
||||
panels.append({
|
||||
"id": 20,
|
||||
"title": "Container Network Receive (RX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 21,
|
||||
"title": "Container Network Transmit (TX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
dash["panels"] = panels
|
||||
return dash
|
||||
|
||||
clusters = [
|
||||
{"key": "aerodrome", "title": "Aerodrome", "folder_id": 5, "folder_uid": "folder-aerodrome", "emoji": "✈️"},
|
||||
{"key": "spaceport", "title": "Spaceport", "folder_id": 6, "folder_uid": "folder-spaceport", "emoji": "🚀"},
|
||||
{"key": "shipyard", "title": "Shipyard", "folder_id": 7, "folder_uid": "folder-shipyard", "emoji": "⚓"}
|
||||
]
|
||||
|
||||
# 1. Generate and Upload Dashboards
|
||||
for c in clusters:
|
||||
dash_json = build_cluster_dashboard(c["key"], c["title"], c["folder_id"], c["folder_uid"], c["emoji"])
|
||||
|
||||
# Save to local gitops repo
|
||||
filename = f"homelab-gitops-repo/dashboards/grafana/{c['key']}-resource-monitor.json"
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
json.dump(dash_json, f, indent=2)
|
||||
print(f"Saved {filename}")
|
||||
|
||||
# Upload to Grafana API
|
||||
payload = {
|
||||
"dashboard": dash_json,
|
||||
"folderId": c["folder_id"],
|
||||
"folderUid": c["folder_uid"],
|
||||
"overwrite": True
|
||||
}
|
||||
resp = requests.post(f"{base_url}/api/dashboards/db", json=payload, auth=auth, verify=False, timeout=10)
|
||||
print(f"Uploaded {c['title']} dashboard to Grafana: {resp.status_code} - {resp.json().get('status')}")
|
||||
|
||||
# 2. Delete old dashboards from AI Stack folder
|
||||
for old_uid in ["snarfnet-systems", "snarfnet-containers"]:
|
||||
del_resp = requests.delete(f"{base_url}/api/dashboards/uid/{old_uid}", auth=auth, verify=False, timeout=10)
|
||||
print(f"Deleted old dashboard {old_uid}: {del_resp.status_code}")
|
||||
|
||||
print("\nDashboard generation and deployment complete!")
|
||||
|
||||
Reference in New Issue
Block a user