247 lines
11 KiB
Python
247 lines
11 KiB
Python
import json
|
|
|
|
def make_panel_stat(id_val, title, expr, grid_pos, unit="none", color="green"):
|
|
return {
|
|
"id": id_val,
|
|
"title": title,
|
|
"type": "stat",
|
|
"gridPos": grid_pos,
|
|
"datasource": "Prometheus",
|
|
"targets": [{"expr": expr, "legendFormat": "{{instance}}", "refId": "A"}],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": color, "value": None}]},
|
|
"unit": unit
|
|
}
|
|
},
|
|
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
|
}
|
|
|
|
def make_panel_timeseries(id_val, title, targets, grid_pos, unit="none"):
|
|
return {
|
|
"id": id_val,
|
|
"title": title,
|
|
"type": "timeseries",
|
|
"gridPos": grid_pos,
|
|
"datasource": "Prometheus",
|
|
"targets": targets,
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 2,
|
|
"fillOpacity": 15
|
|
},
|
|
"unit": unit
|
|
}
|
|
},
|
|
"options": {"tooltip": {"mode": "multi", "sort": "desc"}}
|
|
}
|
|
|
|
def make_panel_bargauge(id_val, title, expr, grid_pos, unit="percent"):
|
|
return {
|
|
"id": id_val,
|
|
"title": title,
|
|
"type": "bargauge",
|
|
"gridPos": grid_pos,
|
|
"datasource": "Prometheus",
|
|
"targets": [{"expr": expr, "legendFormat": "{{instance}}", "refId": "A"}],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": None},
|
|
{"color": "yellow", "value": 75},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": unit,
|
|
"min": 0,
|
|
"max": 100
|
|
}
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"showUnfilled": True
|
|
}
|
|
}
|
|
|
|
# ==================== 1. SYSTEMS DASHBOARD ====================
|
|
systems_dashboard = {
|
|
"annotations": {"list": []},
|
|
"editable": True,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": None,
|
|
"links": [],
|
|
"liveNow": False,
|
|
"panels": [
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
|
"id": 100,
|
|
"title": "🖥️ Systems Fleet Overview & Capacity",
|
|
"type": "row"
|
|
},
|
|
make_panel_stat(1, "Online Node Exporters", "count(node_load1)", {"h": 4, "w": 6, "x": 0, "y": 1}, "none", "green"),
|
|
make_panel_stat(2, "Avg Cluster CPU Load (1m)", "avg(node_load1)", {"h": 4, "w": 6, "x": 6, "y": 1}, "none", "blue"),
|
|
make_panel_stat(3, "Total Monitored RAM", "sum(node_memory_MemTotal_bytes)", {"h": 4, "w": 6, "x": 12, "y": 1}, "bytes", "purple"),
|
|
make_panel_stat(4, "Fleet RAM In-Use %", "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", {"h": 4, "w": 6, "x": 18, "y": 1}, "percent", "orange"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 5},
|
|
"id": 200,
|
|
"title": "⚡ CPU Load & Core Utilization",
|
|
"type": "row"
|
|
},
|
|
make_panel_timeseries(5, "CPU Utilization by Host (%)", [
|
|
{"expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 0, "y": 6}, "percent"),
|
|
make_panel_timeseries(6, "1-Minute Load Average by Host", [
|
|
{"expr": "node_load1", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 12, "y": 6}, "none"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 14},
|
|
"id": 300,
|
|
"title": "🧠 Memory Consumption & Free Capacity",
|
|
"type": "row"
|
|
},
|
|
make_panel_timeseries(7, "Memory Used % by Host", [
|
|
{"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 0, "y": 15}, "percent"),
|
|
make_panel_timeseries(8, "Available RAM by Host (GiB)", [
|
|
{"expr": "node_memory_MemAvailable_bytes", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 12, "y": 15}, "bytes"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 23},
|
|
"id": 400,
|
|
"title": "💾 Filesystem & Disk I/O Performance",
|
|
"type": "row"
|
|
},
|
|
make_panel_bargauge(9, "Root (/) Filesystem Usage %", "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100", {"h": 8, "w": 8, "x": 0, "y": 24}, "percent"),
|
|
make_panel_timeseries(10, "Disk Read Throughput (Bytes/sec)", [
|
|
{"expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))", "legendFormat": "Read {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 8, "x": 8, "y": 24}, "Bps"),
|
|
make_panel_timeseries(11, "Disk Write Throughput (Bytes/sec)", [
|
|
{"expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))", "legendFormat": "Write {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 8, "x": 16, "y": 24}, "Bps"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 32},
|
|
"id": 500,
|
|
"title": "🌐 Network Throughput (RX / TX)",
|
|
"type": "row"
|
|
},
|
|
make_panel_timeseries(12, "Host Network Receive (RX)", [
|
|
{"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", "legendFormat": "RX {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 0, "y": 33}, "Bps"),
|
|
make_panel_timeseries(13, "Host Network Transmit (TX)", [
|
|
{"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", "legendFormat": "TX {{instance}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 12, "y": 33}, "Bps")
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 39,
|
|
"tags": ["infrastructure", "systems", "snarfnet", "node-exporter"],
|
|
"templating": {"list": []},
|
|
"time": {"from": "now-1h", "to": "now"},
|
|
"timepicker": {},
|
|
"timezone": "browser",
|
|
"title": "🖥️ Systems & Cluster Host Performance",
|
|
"uid": "snarfnet-systems"
|
|
}
|
|
|
|
with open("homelab-gitops-repo/manifests/aerodrome/grafana/systems-dashboard.json", "w", encoding="utf-8") as f:
|
|
json.dump(systems_dashboard, f, indent=2)
|
|
|
|
print("Created systems-dashboard.json successfully!")
|
|
|
|
# ==================== 2. CONTAINER METRICS DASHBOARD ====================
|
|
container_dashboard = {
|
|
"annotations": {"list": []},
|
|
"editable": True,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": None,
|
|
"links": [],
|
|
"liveNow": False,
|
|
"panels": [
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
|
"id": 100,
|
|
"title": "📦 Container Fleet & Pod Metrics",
|
|
"type": "row"
|
|
},
|
|
make_panel_stat(1, "Monitored Containers", "count(container_last_seen{container!=''})", {"h": 4, "w": 6, "x": 0, "y": 1}, "none", "green"),
|
|
make_panel_stat(2, "Active Kubernetes Pods", "count(count by (pod, namespace) (container_last_seen{container!=''}))", {"h": 4, "w": 6, "x": 6, "y": 1}, "none", "blue"),
|
|
make_panel_stat(3, "Total Container CPU Load", "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))", {"h": 4, "w": 6, "x": 12, "y": 1}, "none", "purple"),
|
|
make_panel_stat(4, "Total Container RAM", "sum(container_memory_working_set_bytes{container!=''})", {"h": 4, "w": 6, "x": 18, "y": 1}, "bytes", "orange"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 5},
|
|
"id": 200,
|
|
"title": "⚡ Top Container CPU Utilization",
|
|
"type": "row"
|
|
},
|
|
make_panel_timeseries(5, "Top 10 CPU Consuming Pods (Cores)", [
|
|
{"expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))", "legendFormat": "{{namespace}} / {{pod}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 0, "y": 6}, "none"),
|
|
make_panel_timeseries(6, "CPU Throttling by Namespace", [
|
|
{"expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))", "legendFormat": "{{namespace}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 12, "y": 6}, "none"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 14},
|
|
"id": 300,
|
|
"title": "🧠 Container Memory Working Set",
|
|
"type": "row"
|
|
},
|
|
make_panel_timeseries(7, "Top 10 Memory Consuming Pods", [
|
|
{"expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))", "legendFormat": "{{namespace}} / {{pod}}", "refId": "A"}
|
|
], {"h": 8, "w": 14, "x": 0, "y": 15}, "bytes"),
|
|
make_panel_bargauge(8, "Memory by Namespace", "sum by (namespace) (container_memory_working_set_bytes{container!=''})", {"h": 8, "w": 10, "x": 14, "y": 15}, "bytes"),
|
|
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 23},
|
|
"id": 400,
|
|
"title": "🌐 Container Network Traffic",
|
|
"type": "row"
|
|
},
|
|
make_panel_timeseries(9, "Top Pod Network RX (Receive)", [
|
|
{"expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))", "legendFormat": "RX {{namespace}}/{{pod}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 0, "y": 24}, "Bps"),
|
|
make_panel_timeseries(10, "Top Pod Network TX (Transmit)", [
|
|
{"expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))", "legendFormat": "TX {{namespace}}/{{pod}}", "refId": "A"}
|
|
], {"h": 8, "w": 12, "x": 12, "y": 24}, "Bps")
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 39,
|
|
"tags": ["kubernetes", "containers", "cadvisor", "snarfnet", "workloads"],
|
|
"templating": {"list": []},
|
|
"time": {"from": "now-1h", "to": "now"},
|
|
"timepicker": {},
|
|
"timezone": "browser",
|
|
"title": "📦 Kubernetes Containers & Workloads",
|
|
"uid": "snarfnet-containers"
|
|
}
|
|
|
|
with open("homelab-gitops-repo/manifests/aerodrome/grafana/container-metrics-dashboard.json", "w", encoding="utf-8") as f:
|
|
json.dump(container_dashboard, f, indent=2)
|
|
|
|
print("Created container-metrics-dashboard.json successfully!")
|
|
|