280 lines
8.9 KiB
Python
280 lines
8.9 KiB
Python
import json
|
|
|
|
dashboard = {
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": "-- Grafana --",
|
|
"enable": True,
|
|
"hide": True,
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": True,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": None,
|
|
"links": [],
|
|
"liveNow": False,
|
|
"panels": [
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
|
"id": 100,
|
|
"title": "⚡ AI Stack Executive Overview & Cloud Cost Offsets",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"id": 1,
|
|
"title": "Prompt Tokens Processed (In)",
|
|
"type": "stat",
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "sum(vllm:prompt_tokens_total)", "legendFormat": "Prompt Tokens", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": None }] },
|
|
"unit": "locale"
|
|
}
|
|
},
|
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" }
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Generation Tokens Synthesized (Out)",
|
|
"type": "stat",
|
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "sum(vllm:generation_tokens_total)", "legendFormat": "Gen Tokens", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": None }] },
|
|
"unit": "locale"
|
|
}
|
|
},
|
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" }
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Homelab Cloud API Savings ($USD)",
|
|
"type": "stat",
|
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", "legendFormat": "Net Savings", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": None }] },
|
|
"unit": "currencyUSD"
|
|
}
|
|
},
|
|
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center" }
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Active GPU Cluster Health",
|
|
"type": "stat",
|
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "count(DCGM_FI_DEV_GPU_TEMP)", "legendFormat": "Tesla T4s Active", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": None }] },
|
|
"unit": "short"
|
|
}
|
|
},
|
|
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center" }
|
|
},
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
|
"id": 200,
|
|
"title": "🎮 Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "GPU Core Utilization (%)",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "{{Hostname}} ({{modelName}})", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20 },
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "VRAM Memory Allocated & Used",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Used", "refId": "A" },
|
|
{ "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Total", "refId": "B" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 },
|
|
"unit": "bytes"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "GPU Thermal Profile (°C)",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "{{Hostname}} (°C)", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 },
|
|
"unit": "celsius",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": None },
|
|
{ "color": "yellow", "value": 65 },
|
|
{ "color": "red", "value": 80 }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "GPU Power Draw (Watts)",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "{{Hostname}} (Watts)", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 },
|
|
"unit": "watt"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"collapsed": False,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
|
|
"id": 300,
|
|
"title": "🧠 vLLM Real-Time Inference Dynamics & Token Velocity",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "Token Generation Velocity (Tokens / Sec)",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (Out)", "refId": "A" },
|
|
{ "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (In)", "refId": "B" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 25 },
|
|
"unit": "cps"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "Inference Concurrency & Request Backlog",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "vllm:num_requests_running", "legendFormat": "{{model_name}} Running", "refId": "A" },
|
|
{ "expr": "vllm:num_requests_waiting", "legendFormat": "{{model_name}} Queued", "refId": "B" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "stepAfter", "fillOpacity": 15 },
|
|
"unit": "short"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "GPU KV Cache Memory Saturation",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "vllm:gpu_cache_usage_factor * 100", "legendFormat": "{{model_name}} Cache Usage", "refId": "A" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 30 },
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"title": "NVIDIA Triton & Qdrant Operations",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 },
|
|
"datasource": "Prometheus",
|
|
"targets": [
|
|
{ "expr": "rate(nv_inference_request_success[2m])", "legendFormat": "Triton Success/s", "refId": "A" },
|
|
{ "expr": "cpu_cores_used", "legendFormat": "Qdrant CPU Cores", "refId": "B" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth" },
|
|
"unit": "short"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 39,
|
|
"tags": ["ai", "vllm", "dcgm", "gpu", "snarfnet"],
|
|
"templating": { "list": [] },
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"timepicker": {
|
|
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"]
|
|
},
|
|
"timezone": "browser",
|
|
"title": "SnarfNet AI Stack & Inference Intelligence",
|
|
"uid": "snarfnet-ai-stack",
|
|
"version": 1
|
|
}
|
|
|
|
with open("homelab-gitops-repo/manifests/aerodrome/grafana/ai-stack-dashboard.json", "w") as f:
|
|
json.dump(dashboard, f, indent=2)
|
|
|
|
print("Created ai-stack-dashboard.json successfully!")
|
|
|