feat: add homelab deployment configs, documentation, and automation scripts
This commit is contained in:
279
scripts/generate_ai_dashboard.py
Normal file
279
scripts/generate_ai_dashboard.py
Normal file
@@ -0,0 +1,279 @@
|
||||
import json
|
||||
|
||||
dashboard = {
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": True,
|
||||
"hide": True,
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": True,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": None,
|
||||
"links": [],
|
||||
"liveNow": False,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": False,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 100,
|
||||
"title": "⚡ AI Stack Executive Overview & Cloud Cost Offsets",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Prompt Tokens Processed (In)",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "sum(vllm:prompt_tokens_total)", "legendFormat": "Prompt Tokens", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": None }] },
|
||||
"unit": "locale"
|
||||
}
|
||||
},
|
||||
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" }
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Generation Tokens Synthesized (Out)",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "sum(vllm:generation_tokens_total)", "legendFormat": "Gen Tokens", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": None }] },
|
||||
"unit": "locale"
|
||||
}
|
||||
},
|
||||
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" }
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Homelab Cloud API Savings ($USD)",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", "legendFormat": "Net Savings", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": None }] },
|
||||
"unit": "currencyUSD"
|
||||
}
|
||||
},
|
||||
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center" }
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Active GPU Cluster Health",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "count(DCGM_FI_DEV_GPU_TEMP)", "legendFormat": "Tesla T4s Active", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": None }] },
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center" }
|
||||
},
|
||||
{
|
||||
"collapsed": False,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 200,
|
||||
"title": "🎮 Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "GPU Core Utilization (%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "{{Hostname}} ({{modelName}})", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20 },
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "VRAM Memory Allocated & Used",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Used", "refId": "A" },
|
||||
{ "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Total", "refId": "B" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 },
|
||||
"unit": "bytes"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "GPU Thermal Profile (°C)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "{{Hostname}} (°C)", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 },
|
||||
"unit": "celsius",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": None },
|
||||
{ "color": "yellow", "value": 65 },
|
||||
{ "color": "red", "value": 80 }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "GPU Power Draw (Watts)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "{{Hostname}} (Watts)", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 },
|
||||
"unit": "watt"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": False,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
|
||||
"id": 300,
|
||||
"title": "🧠 vLLM Real-Time Inference Dynamics & Token Velocity",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Token Generation Velocity (Tokens / Sec)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (Out)", "refId": "A" },
|
||||
{ "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (In)", "refId": "B" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 25 },
|
||||
"unit": "cps"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Inference Concurrency & Request Backlog",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "vllm:num_requests_running", "legendFormat": "{{model_name}} Running", "refId": "A" },
|
||||
{ "expr": "vllm:num_requests_waiting", "legendFormat": "{{model_name}} Queued", "refId": "B" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "stepAfter", "fillOpacity": 15 },
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "GPU KV Cache Memory Saturation",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "vllm:gpu_cache_usage_factor * 100", "legendFormat": "{{model_name}} Cache Usage", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 30 },
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "NVIDIA Triton & Qdrant Operations",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 },
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{ "expr": "rate(nv_inference_request_success[2m])", "legendFormat": "Triton Success/s", "refId": "A" },
|
||||
{ "expr": "cpu_cores_used", "legendFormat": "Qdrant CPU Cores", "refId": "B" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "drawStyle": "line", "lineInterpolation": "smooth" },
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["ai", "vllm", "dcgm", "gpu", "snarfnet"],
|
||||
"templating": { "list": [] },
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"timepicker": {
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "SnarfNet AI Stack & Inference Intelligence",
|
||||
"uid": "snarfnet-ai-stack",
|
||||
"version": 1
|
||||
}
|
||||
|
||||
with open("homelab-gitops-repo/manifests/aerodrome/grafana/ai-stack-dashboard.json", "w") as f:
|
||||
json.dump(dashboard, f, indent=2)
|
||||
|
||||
print("Created ai-stack-dashboard.json successfully!")
|
||||
|
||||
Reference in New Issue
Block a user