Compare commits
27 Commits
ed38845802
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 5169a02440 | |||
| 7381ddafd4 | |||
| c7eb47f6d8 | |||
| 4ef2b40295 | |||
| 63ab19172d | |||
| a155a495fe | |||
| 4297df615c | |||
| d45c736dcd | |||
| 9cbc274033 | |||
| 620300f56f | |||
| 8b8fdeaf51 | |||
| 911dfd2200 | |||
| bf6aa90c63 | |||
| 27d7ccdfdf | |||
| efecce33db | |||
| dfa2578723 | |||
| 28f199f604 | |||
| bac981091c | |||
| e65149381e | |||
| 3ddc7d01ab | |||
| 96e6b7a9b9 | |||
| 9f0c3756ac | |||
| 7967eead8e | |||
| 79e2032603 | |||
| 7b2dd24526 | |||
| be820c6b92 | |||
| a7c636cac0 |
28
argocd/aerodrome/prometheus-app.yaml
Normal file
28
argocd/aerodrome/prometheus-app.yaml
Normal file
@@ -0,0 +1,28 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: aerodrome-prometheus
|
||||
namespace: infrastructure
|
||||
labels:
|
||||
app.kubernetes.io/name: aerodrome-prometheus
|
||||
cluster: aerodrome
|
||||
annotations:
|
||||
notifications.argoproj.io/subscribe.on-sync-failed.gotify: ""
|
||||
notifications.argoproj.io/subscribe.on-health-degraded.gotify: ""
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: "https://git.snarfnet.net/Snarf/homelab-gitops.git"
|
||||
targetRevision: HEAD
|
||||
path: manifests/aerodrome/prometheus
|
||||
destination:
|
||||
server: "https://140.44.4.11:6443"
|
||||
namespace: default
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: false
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
|
||||
523
dashboards/grafana/ai-stack-dashboard.json
Normal file
523
dashboards/grafana/ai-stack-dashboard.json
Normal file
@@ -0,0 +1,523 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 100,
|
||||
"title": "\u26a1 AI Stack Executive Overview & Cloud Cost Offsets",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Prompt Tokens Processed (In)",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(vllm:prompt_tokens_total)",
|
||||
"legendFormat": "Prompt Tokens",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "blue",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "locale"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Generation Tokens Synthesized (Out)",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(vllm:generation_tokens_total)",
|
||||
"legendFormat": "Gen Tokens",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "locale"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Homelab Cloud API Savings ($USD)",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)",
|
||||
"legendFormat": "Net Savings",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "currencyUSD"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Active GPU Cluster Health",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(DCGM_FI_DEV_GPU_TEMP)",
|
||||
"legendFormat": "Tesla T4s Active",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "purple",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 200,
|
||||
"title": "\ud83c\udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "GPU Core Utilization (%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "DCGM_FI_DEV_GPU_UTIL",
|
||||
"legendFormat": "{{Hostname}} ({{modelName}})",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "VRAM Memory Allocated & Used",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024",
|
||||
"legendFormat": "{{Hostname}} VRAM Used",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024",
|
||||
"legendFormat": "{{Hostname}} VRAM Total",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "GPU Thermal Profile (\u00b0C)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "DCGM_FI_DEV_GPU_TEMP",
|
||||
"legendFormat": "{{Hostname}} (\u00b0C)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10
|
||||
},
|
||||
"unit": "celsius",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 65
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "GPU Power Draw (Watts)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 14
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "DCGM_FI_DEV_POWER_USAGE",
|
||||
"legendFormat": "{{Hostname}} (Watts)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "watt"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 21
|
||||
},
|
||||
"id": 300,
|
||||
"title": "\ud83e\udde0 vLLM Real-Time Inference Dynamics & Token Velocity",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Token Generation Velocity (Tokens / Sec)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 22
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)",
|
||||
"legendFormat": "{{model_name}} (Out)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)",
|
||||
"legendFormat": "{{model_name}} (In)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 25
|
||||
},
|
||||
"unit": "cps"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Inference Concurrency & Request Backlog",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 22
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "vllm:num_requests_running",
|
||||
"legendFormat": "{{model_name}} Running",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "vllm:num_requests_waiting",
|
||||
"legendFormat": "{{model_name}} Queued",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "stepAfter",
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "GPU KV Cache Memory Saturation",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 30
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "vllm:gpu_cache_usage_factor * 100",
|
||||
"legendFormat": "{{model_name}} Cache Usage",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 30
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "NVIDIA Triton & Qdrant Operations",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 30
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(nv_inference_request_success[2m])",
|
||||
"legendFormat": "Triton Success/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "cpu_cores_used",
|
||||
"legendFormat": "Qdrant CPU Cores",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth"
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"ai",
|
||||
"vllm",
|
||||
"dcgm",
|
||||
"gpu",
|
||||
"snarfnet"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "SnarfNet AI Stack & Inference Intelligence",
|
||||
"uid": "snarfnet-ai-stack",
|
||||
"version": 1
|
||||
}
|
||||
476
dashboards/grafana/container-metrics-dashboard.json
Normal file
476
dashboards/grafana/container-metrics-dashboard.json
Normal file
@@ -0,0 +1,476 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 100,
|
||||
"title": "\ud83d\udce6 Container Fleet & Pod Metrics",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Monitored Containers",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(container_last_seen{container!=''})",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Active Kubernetes Pods",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(count by (pod, namespace) (container_last_seen{container!=''}))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "blue",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Total Container CPU Load",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "purple",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Total Container RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_working_set_bytes{container!=''})",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "orange",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 200,
|
||||
"title": "\u26a1 Top Container CPU Utilization",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Top 10 CPU Consuming Pods (Cores)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))",
|
||||
"legendFormat": "{{namespace}} / {{pod}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "CPU Throttling by Namespace",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))",
|
||||
"legendFormat": "{{namespace}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"id": 300,
|
||||
"title": "\ud83e\udde0 Container Memory Working Set",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Top 10 Memory Consuming Pods",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 14,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))",
|
||||
"legendFormat": "{{namespace}} / {{pod}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Memory by Namespace",
|
||||
"type": "bargauge",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 10,
|
||||
"x": 14,
|
||||
"y": 15
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (namespace) (container_memory_working_set_bytes{container!=''})",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"showUnfilled": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 23
|
||||
},
|
||||
"id": 400,
|
||||
"title": "\ud83c\udf10 Container Network Traffic",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Top Pod Network RX (Receive)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))",
|
||||
"legendFormat": "RX {{namespace}}/{{pod}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "Bps"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Top Pod Network TX (Transmit)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 24
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))",
|
||||
"legendFormat": "TX {{namespace}}/{{pod}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "Bps"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"kubernetes",
|
||||
"containers",
|
||||
"cadvisor",
|
||||
"snarfnet",
|
||||
"workloads"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "\ud83d\udce6 Kubernetes Containers & Workloads",
|
||||
"uid": "snarfnet-containers"
|
||||
}
|
||||
595
dashboards/grafana/systems-dashboard.json
Normal file
595
dashboards/grafana/systems-dashboard.json
Normal file
@@ -0,0 +1,595 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 100,
|
||||
"title": "\ud83d\udda5\ufe0f Systems Fleet Overview & Capacity",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Online Node Exporters",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(node_load1)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Avg Cluster CPU Load (1m)",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg(node_load1)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "blue",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Total Monitored RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(node_memory_MemTotal_bytes)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "purple",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Fleet RAM In-Use %",
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "orange",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 200,
|
||||
"title": "\u26a1 CPU Load & Core Utilization",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "CPU Utilization by Host (%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)",
|
||||
"legendFormat": "{{cluster}} - {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "1-Minute Load Average by Host",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_load1",
|
||||
"legendFormat": "{{cluster}} - {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"id": 300,
|
||||
"title": "\ud83e\udde0 Memory Consumption & Free Capacity",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Memory Used % by Host",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||
"legendFormat": "{{cluster}} - {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Available RAM by Host (GiB)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 15
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_memory_MemAvailable_bytes",
|
||||
"legendFormat": "{{cluster}} - {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 23
|
||||
},
|
||||
"id": 400,
|
||||
"title": "\ud83d\udcbe Filesystem & Disk I/O Performance",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Root (/) Filesystem Usage %",
|
||||
"type": "bargauge",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"showUnfilled": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Disk Read Throughput (Bytes/sec)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 24
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))",
|
||||
"legendFormat": "Read {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "Bps"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Disk Write Throughput (Bytes/sec)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 24
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))",
|
||||
"legendFormat": "Write {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "Bps"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
},
|
||||
"id": 500,
|
||||
"title": "\ud83c\udf10 Network Throughput (RX / TX)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Host Network Receive (RX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 33
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))",
|
||||
"legendFormat": "RX {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "Bps"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Host Network Transmit (TX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 33
|
||||
},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))",
|
||||
"legendFormat": "TX {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"unit": "Bps"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"infrastructure",
|
||||
"systems",
|
||||
"snarfnet",
|
||||
"node-exporter"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "\ud83d\udda5\ufe0f Systems & Cluster Host Performance",
|
||||
"uid": "snarfnet-systems"
|
||||
}
|
||||
1029
docs/snarfnet_homelab_topology.html
Normal file
1029
docs/snarfnet_homelab_topology.html
Normal file
File diff suppressed because it is too large
Load Diff
BIN
docs/snarfnet_homelab_topology.png
Normal file
BIN
docs/snarfnet_homelab_topology.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 830 KiB |
3
manifests/aerodrome/grafana/.argocdignore
Normal file
3
manifests/aerodrome/grafana/.argocdignore
Normal file
@@ -0,0 +1,3 @@
|
||||
*.json
|
||||
*.md
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
281
manifests/aerodrome/prometheus/prometheus.yaml
Normal file
281
manifests/aerodrome/prometheus/prometheus.yaml
Normal file
@@ -0,0 +1,281 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: default
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ''
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- nodes/metrics
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- nonResourceURLs:
|
||||
- /metrics
|
||||
- /metrics/cadvisor
|
||||
verbs:
|
||||
- get
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: prometheus
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: prometheus
|
||||
namespace: default
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: prometheus-storage
|
||||
namespace: default
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: prometheus-config
|
||||
namespace: default
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
data:
|
||||
prometheus.yml: |
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 10s
|
||||
evaluation_interval: 15s
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost:9090
|
||||
- job_name: vllm-qwen
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.84:30800
|
||||
labels:
|
||||
model: Qwen2.5-7B-Instruct
|
||||
node: snaisvr44hrvstr04
|
||||
role: general-instruction
|
||||
- job_name: vllm-gemma4
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.82:30804
|
||||
labels:
|
||||
model: Gemma-4-12B-it
|
||||
node: snaisvr44hrvstr02
|
||||
role: coding-agent
|
||||
- job_name: vllm-deepseek-r1
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.81:30805
|
||||
labels:
|
||||
model: DeepSeek-R1-Distill-Qwen-14B
|
||||
node: snaisvr44hrvstr01
|
||||
role: reasoning-planner
|
||||
- job_name: triton-inference-server
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.83:30803
|
||||
labels:
|
||||
app: triton-inference-server
|
||||
node: snaisvr44hrvstr03
|
||||
model: decision_engine
|
||||
- job_name: qdrant
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.82:30633
|
||||
labels:
|
||||
app: qdrant
|
||||
node: snaisvr44hrvstr02
|
||||
- job_name: nvidia-dcgm
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.81:9400
|
||||
labels:
|
||||
node: snaisvr44hrvstr01
|
||||
gpu_model: Tesla T4
|
||||
- targets:
|
||||
- 140.44.4.82:9400
|
||||
labels:
|
||||
node: snaisvr44hrvstr02
|
||||
gpu_model: Tesla T4
|
||||
- targets:
|
||||
- 140.44.4.83:9400
|
||||
labels:
|
||||
node: snaisvr44hrvstr03
|
||||
gpu_model: Tesla T4
|
||||
- targets:
|
||||
- 140.44.4.84:9400
|
||||
labels:
|
||||
node: snaisvr44hrvstr04
|
||||
gpu_model: Tesla T4
|
||||
- job_name: node-exporter
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 140.44.4.11:9100
|
||||
- 140.44.4.12:9100
|
||||
- 140.44.4.13:9100
|
||||
- 140.44.4.21:9100
|
||||
- 140.44.4.22:9100
|
||||
- 140.44.4.23:9100
|
||||
- 140.44.4.24:9100
|
||||
- 140.44.4.25:9100
|
||||
- 140.44.4.26:9100
|
||||
- 140.44.4.27:9100
|
||||
labels:
|
||||
cluster: aerodrome
|
||||
- targets:
|
||||
- 140.44.4.41:9100
|
||||
- 140.44.4.42:9100
|
||||
- 140.44.4.43:9100
|
||||
- 140.44.4.51:9100
|
||||
- 140.44.4.52:9100
|
||||
- 140.44.4.53:9100
|
||||
- 140.44.4.54:9100
|
||||
- 140.44.4.55:9100
|
||||
- 140.44.4.56:9100
|
||||
- 140.44.4.57:9100
|
||||
labels:
|
||||
cluster: spaceport
|
||||
- job_name: kubernetes-cadvisor
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels:
|
||||
- __meta_kubernetes_node_name
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: default
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 65534
|
||||
runAsUser: 65534
|
||||
runAsNonRoot: true
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: prom/prometheus:v2.54.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=90d
|
||||
- --web.enable-lifecycle
|
||||
- --web.console.libraries=/usr/share/prometheus/console_libraries
|
||||
- --web.console.templates=/usr/share/prometheus/consoles
|
||||
ports:
|
||||
- name: web
|
||||
containerPort: 9090
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: '2'
|
||||
memory: 4Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/ready
|
||||
port: 9090
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /-/healthy
|
||||
port: 9090
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/prometheus
|
||||
- name: storage
|
||||
mountPath: /prometheus
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: prometheus-config
|
||||
- name: storage
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-storage
|
||||
serviceAccountName: prometheus
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: default
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: prometheus
|
||||
ports:
|
||||
- name: web
|
||||
port: 9090
|
||||
targetPort: 9090
|
||||
@@ -115,6 +115,7 @@ kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
traefik.ingress.kubernetes.io/router.middlewares: monitoring-authentik-forwardauth@kubernetescrd
|
||||
traefik.ingress.kubernetes.io/service.serversscheme: http
|
||||
labels:
|
||||
app.kubernetes.io/name: uptime-kuma
|
||||
name: uptime-ing
|
||||
@@ -127,9 +128,9 @@ spec:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: uptime-lb
|
||||
name: uptime-kuma-2-1756668961
|
||||
port:
|
||||
number: 448
|
||||
number: 3001
|
||||
path: /
|
||||
pathType: ImplementationSpecific
|
||||
tls:
|
||||
|
||||
@@ -126,6 +126,8 @@ spec:
|
||||
labels:
|
||||
app: vllm-openai
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: snaisvr44hrvstr04
|
||||
containers:
|
||||
- name: vllm
|
||||
image: vllm/vllm-openai:v0.7.3
|
||||
@@ -136,11 +138,11 @@ spec:
|
||||
- "--served-model-name"
|
||||
- "Qwen2.5-7B-Instruct"
|
||||
- "--quantization"
|
||||
- "awq_marlin"
|
||||
- "awq"
|
||||
- "--dtype"
|
||||
- "half"
|
||||
- "--max-model-len"
|
||||
- "16384"
|
||||
- "65536"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.90"
|
||||
- "--enable-auto-tool-choice"
|
||||
@@ -155,6 +157,8 @@ spec:
|
||||
value: "/root/.cache/huggingface"
|
||||
- name: VLLM_WORKER_MULTIPROC_METHOD
|
||||
value: "spawn"
|
||||
- name: VLLM_ALLOW_LONG_MAX_MODEL_LEN
|
||||
value: "1"
|
||||
ports:
|
||||
- name: http-openai
|
||||
containerPort: 8000
|
||||
@@ -205,6 +209,234 @@ spec:
|
||||
nodePort: 30800
|
||||
---
|
||||
# ==============================================================================
|
||||
# 2B. vLLM GEMMA 4 12B INFERENCE ENGINE (snaisvr44hrvstr02 Tesla T4 GPU)
|
||||
# ==============================================================================
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vllm-gemma4-models-pvc
|
||||
namespace: ai-stack
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vllm-gemma4
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: vllm-gemma4
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: vllm-gemma4
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: vllm-gemma4
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: snaisvr44hrvstr02
|
||||
containers:
|
||||
- name: vllm
|
||||
image: vllm/vllm-openai:v0.32.0-cu129
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
rm -rf /usr/local/lib/python3.12/dist-packages/torchcodec*
|
||||
python3 -c '
|
||||
p = "/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/triton_unified_attention.py"
|
||||
s = open(p).read()
|
||||
s = s.replace("return 32", "return 16")
|
||||
s = s.replace("launch_num_stages: int | None = None", "launch_num_stages: int | None = 1")
|
||||
open(p, "w").write(s)
|
||||
'
|
||||
exec vllm serve google/gemma-4-12B-it-qat-w4a16-ct \
|
||||
--served-model-name Gemma-4-12B-it \
|
||||
--dtype half \
|
||||
--max-model-len 8192 \
|
||||
--gpu-memory-utilization 0.92 \
|
||||
--enforce-eager \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser gemma4 \
|
||||
--reasoning-parser gemma4 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: "/root/.cache/huggingface"
|
||||
- name: VLLM_WORKER_MULTIPROC_METHOD
|
||||
value: "spawn"
|
||||
- name: VLLM_ALLOW_LONG_MAX_MODEL_LEN
|
||||
value: "1"
|
||||
ports:
|
||||
- name: http-openai
|
||||
containerPort: 8000
|
||||
resources:
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: "24Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "24"
|
||||
memory: "48Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: vllm-gemma4-models
|
||||
mountPath: /root/.cache/huggingface
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: vllm-gemma4-models
|
||||
persistentVolumeClaim:
|
||||
claimName: vllm-gemma4-models-pvc
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vllm-gemma4
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: vllm-gemma4
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
app: vllm-gemma4
|
||||
ports:
|
||||
- name: http-openai
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
nodePort: 30804
|
||||
---
|
||||
# ==============================================================================
|
||||
# 2C. DEEPSEEK-R1-DISTILL-QWEN-14B REASONING ENGINE (snaisvr44hrvstr01 Tesla T4 GPU)
|
||||
# ==============================================================================
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: glm-superexpert-models-pvc
|
||||
namespace: ai-stack
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: glm-superexpert
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: glm-superexpert
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: glm-superexpert
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: glm-superexpert
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: snaisvr44hrvstr01
|
||||
containers:
|
||||
- name: vllm
|
||||
image: vllm/vllm-openai:v0.32.0-cu129
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
rm -rf /usr/local/lib/python3.12/dist-packages/torchcodec*
|
||||
exec vllm serve casperhansen/deepseek-r1-distill-qwen-14b-awq \
|
||||
--served-model-name DeepSeek-R1-Distill-Qwen-14B \
|
||||
--quantization awq \
|
||||
--dtype half \
|
||||
--max-model-len 16384 \
|
||||
--gpu-memory-utilization 0.92 \
|
||||
--enforce-eager \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser hermes \
|
||||
--reasoning-parser deepseek_r1 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: "/root/.cache/huggingface"
|
||||
ports:
|
||||
- name: http-openai
|
||||
containerPort: 8000
|
||||
resources:
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: "20Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "24"
|
||||
memory: "40Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: glm-models
|
||||
mountPath: /root/.cache/huggingface
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: glm-models
|
||||
persistentVolumeClaim:
|
||||
claimName: glm-superexpert-models-pvc
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: glm-superexpert
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: glm-superexpert
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
app: glm-superexpert
|
||||
ports:
|
||||
- name: http-openai
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
nodePort: 30805
|
||||
---
|
||||
# ==============================================================================
|
||||
# 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving)
|
||||
# ==============================================================================
|
||||
apiVersion: v1
|
||||
@@ -377,3 +609,53 @@ spec:
|
||||
port: 8002
|
||||
targetPort: 8002
|
||||
nodePort: 30803
|
||||
|
||||
---
|
||||
# ==============================================================================
|
||||
# 4. NVIDIA DCGM EXPORTER (Tesla T4 Telemetry: Utilization, VRAM, Power, Temp)
|
||||
# ==============================================================================
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: dcgm-exporter
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app.kubernetes.io/name: dcgm-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dcgm-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dcgm-exporter
|
||||
spec:
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- name: dcgm-exporter
|
||||
image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
|
||||
env:
|
||||
- name: DCGM_EXPORTER_LISTEN
|
||||
value: ":9400"
|
||||
- name: DCGM_EXPORTER_KUBERNETES
|
||||
value: "false"
|
||||
- name: LD_LIBRARY_PATH
|
||||
value: "/usr/local/nvidia/lib64"
|
||||
securityContext:
|
||||
privileged: true
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 9400
|
||||
volumeMounts:
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
- name: nvidia-lib
|
||||
mountPath: /usr/local/nvidia/lib64
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: nvidia-lib
|
||||
hostPath:
|
||||
path: /usr/local/nvidia/lib64
|
||||
|
||||
@@ -147,39 +147,6 @@ spec:
|
||||
type: ClusterIP
|
||||
---
|
||||
apiVersion: v1
|
||||
data:
|
||||
bookmarks.yaml: "- Developer:\n - Github:\n - abbr: GH\n href: https://github.com/\n"
|
||||
docker.yaml: ''
|
||||
kubernetes.yaml: 'mode: disable
|
||||
|
||||
'
|
||||
services.yaml: "- Critical Services:\n - Authentik:\n icon: authentik.png\n\
|
||||
\ href: https://authentik.snarfnet.net\n widget:\n type: authentik\n\
|
||||
\ url: https://authentik.snarfnet.net\n fields: [\"users\", \"loginsLast24H\"\
|
||||
, \"failedLoginsLast24H\"]\n key: 7HamapUhYizcNudHDIT5IFy3XEpRgeKgqr1Pw4ub9eTpAjMAzB8Aur1Z6dUU\n\
|
||||
\ - PeaNUT UPS:\n icon: peanut.png\n href: https://portal.snarfnet.net\n\
|
||||
\ widget:\n type: peanut\n url: https://peanut.snarfnet.net\n\
|
||||
\ fields: [\"battery_charge\", \"ups_load\", \"ups_status\"]\n key:\
|
||||
\ server-closet\n - Unifi Controller:\n icon: unifi.png\n href: https://unifi.ui.com\n\
|
||||
\ widget:\n type: unifi\n url: https://140.44.44.1\n username:\
|
||||
\ Snarf\n password: Katzir476!!!\n- Git Ops:\n - Gitea:\n icon: gitea.png\n\
|
||||
\ href: https://git.snarfnet.net\n widget:\n type: gitea\n \
|
||||
\ url: https://git.snarfnet.net\n fields: [\"repositories\", \"notifications\"\
|
||||
, \"issues\", \"pulls\"]\n key: 47c10c54574131c007302b246352f653facb5062\n\
|
||||
- Infrastructure:\n - NGINX proxy manager:\n icon: nginx-proxy-manager\n\
|
||||
\ href: https://nginx.snarfnet.net\n widget:\n type: npm\n \
|
||||
\ url: https://nginx.snarfnet.net\n fields: [\"enabled\", \"disabled\"\
|
||||
, \"total\"]\n username: snarf@snarfnet.net\n password: Katzir476!\n\
|
||||
\ - Uptime Kuma:\n icon: uptime-kuma\n href: https://uptime.snarfnet.net\n\
|
||||
\ widget:\n type: uptimekuma\n url: https://uptime.snarfnet.net\n\
|
||||
\ fields: [\"up\", \"down\", \"uptime\", \"incident\"]\n slug: public\n"
|
||||
settings.yaml: "title: SnarfNet Landing Zone\nheaderStyle: boxedWidgets\ncolor:\
|
||||
\ green\ntheme: dark\nlayout:\n Critical Services:\n Git Ops:\n Infrastructure:\n\
|
||||
showStats: true\nproviders:\n longhorn:\n url: https://longhorn.aerodrome.snarfnet.net\n"
|
||||
widgets.yaml: "- resources:\n backend: resources\n cpu: true\n expanded:\
|
||||
\ true\n memory: true\n- search:\n provider: duckduckgo\n target: _blank\n\
|
||||
- longhorn:\n expanded: true\n total: true\n labels: true\n nodes:\
|
||||
\ true\n"
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
annotations:
|
||||
@@ -193,3 +160,261 @@ metadata:
|
||||
helm.sh/chart: homepage-2.1.0
|
||||
name: homepage-2-1744068158
|
||||
namespace: infrastructure
|
||||
data:
|
||||
bookmarks.yaml: |
|
||||
- Quick Links:
|
||||
- GitHub:
|
||||
- abbr: GH
|
||||
href: https://github.com/
|
||||
- UniFi Site Manager:
|
||||
- abbr: UI
|
||||
href: https://unifi.ui.com/
|
||||
- Cloudflare:
|
||||
- abbr: CF
|
||||
href: https://dash.cloudflare.com/
|
||||
- Grafana Telemetry:
|
||||
- Systems Performance:
|
||||
- abbr: SYS
|
||||
href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861
|
||||
- Container Workloads:
|
||||
- abbr: K8S
|
||||
href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads
|
||||
- AI Stack Intelligence:
|
||||
- abbr: AI
|
||||
href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence
|
||||
- Uptime Status Pages:
|
||||
- Unified Overview:
|
||||
- abbr: ALL
|
||||
href: https://uptime.snarfnet.net/status/default
|
||||
- Core & VIP Fabric:
|
||||
- abbr: VIP
|
||||
href: https://uptime.snarfnet.net/status/core
|
||||
- GitOps & CI/CD:
|
||||
- abbr: OPS
|
||||
href: https://uptime.snarfnet.net/status/gitops
|
||||
- Edge & Ingress:
|
||||
- abbr: EDG
|
||||
href: https://uptime.snarfnet.net/status/edge
|
||||
- Homelab Architecture:
|
||||
- Topology Diagram (Image):
|
||||
- abbr: IMG
|
||||
href: https://homepage.snarfnet.net/topology.png
|
||||
- Interactive Topology (HTML):
|
||||
- abbr: MAP
|
||||
href: https://homepage.snarfnet.net/topology.html
|
||||
docker.yaml: ''
|
||||
kubernetes.yaml: |
|
||||
mode: cluster
|
||||
services.yaml: |
|
||||
- Critical Services:
|
||||
- Authentik:
|
||||
icon: authentik.png
|
||||
href: https://authentik.snarfnet.net
|
||||
description: SSO & Identity Management
|
||||
- PeaNUT UPS:
|
||||
icon: peanut.png
|
||||
href: https://portal.snarfnet.net
|
||||
description: UPS Power & Battery Monitor
|
||||
- Unifi Controller:
|
||||
icon: unifi.png
|
||||
href: https://unifi.ui.com
|
||||
widget:
|
||||
type: unifi
|
||||
url: https://140.44.44.1
|
||||
username: Snarf
|
||||
password: Katzir476!!!
|
||||
- GitOps & Automation:
|
||||
- Gitea:
|
||||
icon: gitea.png
|
||||
href: https://git.snarfnet.net
|
||||
description: Git Repositories
|
||||
widget:
|
||||
type: gitea
|
||||
url: https://git.snarfnet.net
|
||||
key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71
|
||||
- ArgoCD:
|
||||
icon: argo-cd.png
|
||||
href: https://argo.snarfnet.net
|
||||
description: Declarative GitOps Reconciler
|
||||
widget:
|
||||
type: argocd
|
||||
url: https://argo.snarfnet.net
|
||||
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0
|
||||
- AWX Automation:
|
||||
icon: awx.png
|
||||
href: https://awx.snarfnet.net
|
||||
description: Ansible Automation Controller
|
||||
- Semaphore:
|
||||
icon: semaphore.png
|
||||
href: https://semaphore.snarfnet.net
|
||||
description: Playbook Execution Engine
|
||||
- Storage & Clusters:
|
||||
- Longhorn Aerodrome:
|
||||
icon: longhorn.png
|
||||
href: https://longhorn.aerodrome.snarfnet.net
|
||||
description: Aerodrome Distributed Flash Storage
|
||||
- Longhorn Spaceport:
|
||||
icon: longhorn.png
|
||||
href: https://longhorn.spaceport.snarfnet.net
|
||||
description: Spaceport Distributed Block Storage
|
||||
- Rancher (Aerodrome):
|
||||
icon: rancher.png
|
||||
href: https://aerodrome.snarfnet.net
|
||||
description: Kubernetes Management Console
|
||||
widget:
|
||||
type: customapi
|
||||
url: https://aerodrome.snarfnet.net/v3/clusters/local
|
||||
headers:
|
||||
Authorization: Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7
|
||||
mappings:
|
||||
- field: capacity.cpu
|
||||
label: CPU Cores
|
||||
- field: capacity.pods
|
||||
label: Max Pods
|
||||
- field: state
|
||||
label: Cluster State
|
||||
- Harvester HCI (Spacedock):
|
||||
icon: harvester.png
|
||||
href: https://harvester.spacedock.snarfnet.net
|
||||
description: HCI Virtualization Hypervisors
|
||||
widget:
|
||||
type: customapi
|
||||
url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt
|
||||
headers:
|
||||
Authorization: Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7
|
||||
mappings:
|
||||
- field: capacity.cpu
|
||||
label: CPU Cores
|
||||
- field: capacity.pods
|
||||
label: Max Pods
|
||||
- field: state
|
||||
label: HCI State
|
||||
- Portainer Swarm:
|
||||
icon: portainer.png
|
||||
href: https://shipyard.snarfnet.net
|
||||
description: Docker Swarm Container Fleet
|
||||
widget:
|
||||
type: portainer
|
||||
url: https://140.44.4.71:9443
|
||||
env: 3
|
||||
key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8=
|
||||
- Infrastructure & Observability:
|
||||
- NGINX Proxy Manager:
|
||||
icon: nginx-proxy-manager
|
||||
href: https://nginx.snarfnet.net
|
||||
description: Reverse Proxy & SSL Certificates
|
||||
- Uptime Kuma:
|
||||
icon: uptime-kuma
|
||||
href: https://uptime.snarfnet.net
|
||||
description: Multi-Endpoint Availability Probes
|
||||
widget:
|
||||
type: uptimekuma
|
||||
url: http://140.44.4.28:448
|
||||
slug: default
|
||||
- Zabbix:
|
||||
icon: zabbix.png
|
||||
href: https://zabbix.snarfnet.net
|
||||
description: Infrastructure Metrics & Alarms
|
||||
- Grafana:
|
||||
icon: grafana.png
|
||||
href: https://grafana.snarfnet.net
|
||||
description: Real-Time Metrics & AI Dashboards
|
||||
widget:
|
||||
type: grafana
|
||||
url: https://grafana.snarfnet.net
|
||||
username: admin
|
||||
password: Katzir476!!!
|
||||
- Core Node Glances:
|
||||
icon: glances.png
|
||||
href: http://140.44.4.11:61208
|
||||
description: Host Resource Monitor
|
||||
- Homelab Topology:
|
||||
icon: mdi-sitemap
|
||||
href: https://homepage.snarfnet.net/topology.html
|
||||
description: Architecture & Network Schematic
|
||||
- Environment & Weather:
|
||||
- Tempest Weather Station:
|
||||
icon: mdi-weather-partly-cloudy
|
||||
href: https://snarfhome.snarfnet.net
|
||||
description: Backyard Weather Telemetry
|
||||
widget:
|
||||
type: homeassistant
|
||||
url: http://14.4.4.20:8123
|
||||
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4
|
||||
custom:
|
||||
- template: '{{ states(''sensor.st_00158555_temperature'') | float(0) | round(1)
|
||||
}} °F'
|
||||
label: Outdoor Temp
|
||||
- template: '{{ states(''sensor.st_00158555_feels_like'') | float(0) | round(1)
|
||||
}} °F'
|
||||
label: Feels Like
|
||||
- template: '{{ states(''sensor.st_00158555_humidity'') | float(0) | round(1)
|
||||
}} %'
|
||||
label: Humidity
|
||||
- template: '{{ states(''sensor.st_00158555_wind_speed'') | float(0) | round(1)
|
||||
}} mph'
|
||||
label: Wind Speed
|
||||
settings.yaml: |
|
||||
title: SnarfNet Operations & Telemetry
|
||||
headerStyle: boxed
|
||||
color: slate
|
||||
theme: dark
|
||||
statusStyle: badge
|
||||
background:
|
||||
image: https://i.imgur.com/hOzzkCY.jpeg
|
||||
blur: sm
|
||||
opacity: 50
|
||||
brightness: 50
|
||||
cardBlur: sm
|
||||
quicklaunch:
|
||||
searchDescriptions: true
|
||||
hideInternetSearch: true
|
||||
hideVisitURL: true
|
||||
hideVersion: true
|
||||
layout:
|
||||
Critical Services:
|
||||
icon: mdi-shield-check
|
||||
style: row
|
||||
columns: 3
|
||||
GitOps & Automation:
|
||||
icon: mdi-git
|
||||
style: row
|
||||
columns: 4
|
||||
Storage & Clusters:
|
||||
icon: mdi-server-network
|
||||
style: row
|
||||
columns: 3
|
||||
Infrastructure & Observability:
|
||||
icon: mdi-chart-timeline-variant
|
||||
style: row
|
||||
columns: 3
|
||||
Environment & Weather:
|
||||
icon: mdi-weather-partly-cloudy
|
||||
style: row
|
||||
columns: 1
|
||||
showStats: true
|
||||
providers:
|
||||
longhorn:
|
||||
url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500
|
||||
widgets.yaml: |
|
||||
- kubernetes:
|
||||
cluster:
|
||||
show: true
|
||||
cpu: true
|
||||
memory: true
|
||||
show_names: true
|
||||
nodes:
|
||||
show: false
|
||||
- longhorn:
|
||||
total: true
|
||||
nodes: false
|
||||
- resources:
|
||||
label: System
|
||||
backend: resources
|
||||
cpu: true
|
||||
expanded: true
|
||||
memory: true
|
||||
disk: /
|
||||
- search:
|
||||
provider: duckduckgo
|
||||
target: _blank
|
||||
|
||||
34
manifests/spaceport/homepage/rbac.yaml
Normal file
34
manifests/spaceport/homepage/rbac.yaml
Normal file
@@ -0,0 +1,34 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: homepage-reader
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes", "namespaces", "pods", "services", "endpoints"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["ingresses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["nodes", "pods"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: ["apiextensions.k8s.io"]
|
||||
resources: ["customresourcedefinitions"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: ["traefik.containo.us", "traefik.io"]
|
||||
resources: ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "traefikservices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: homepage-reader-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: homepage-reader
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: default
|
||||
namespace: infrastructure
|
||||
|
||||
Reference in New Issue
Block a user