diff --git a/argocd/aerodrome/prometheus-app.yaml b/argocd/aerodrome/prometheus-app.yaml new file mode 100644 index 0000000..40dd927 --- /dev/null +++ b/argocd/aerodrome/prometheus-app.yaml @@ -0,0 +1,27 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: aerodrome-prometheus + namespace: infrastructure + labels: + app.kubernetes.io/name: aerodrome-prometheus + cluster: aerodrome + annotations: + notifications.argoproj.io/subscribe.on-sync-failed.gotify: "" + notifications.argoproj.io/subscribe.on-health-degraded.gotify: "" +spec: + project: homelab + source: + repoURL: "https://git.snarfnet.net/Snarf/homelab-gitops.git" + targetRevision: HEAD + path: manifests/aerodrome/prometheus + destination: + server: "https://140.44.4.11:6443" + namespace: default + syncPolicy: + automated: + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/manifests/aerodrome/grafana/ai-stack-dashboard.json b/manifests/aerodrome/grafana/ai-stack-dashboard.json new file mode 100644 index 0000000..0d0e0ba --- /dev/null +++ b/manifests/aerodrome/grafana/ai-stack-dashboard.json @@ -0,0 +1,523 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\u26a1 AI Stack Executive Overview & Cloud Cost Offsets", + "type": "row" + }, + { + "id": 1, + "title": "Prompt Tokens Processed (In)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(vllm:prompt_tokens_total)", + "legendFormat": "Prompt Tokens", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Generation Tokens Synthesized (Out)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(vllm:generation_tokens_total)", + "legendFormat": "Gen Tokens", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Homelab Cloud API Savings ($USD)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", + "legendFormat": "Net Savings", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + } + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Active GPU Cluster Health", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(DCGM_FI_DEV_GPU_TEMP)", + "legendFormat": "Tesla T4s Active", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + } + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "\ud83c\udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)", + "type": "row" + }, + { + "id": 5, + "title": "GPU Core Utilization (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL", + "legendFormat": "{{Hostname}} ({{modelName}})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + } + }, + { + "id": 6, + "title": "VRAM Memory Allocated & Used", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", + "legendFormat": "{{Hostname}} VRAM Used", + "refId": "A" + }, + { + "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", + "legendFormat": "{{Hostname}} VRAM Total", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15 + }, + "unit": "bytes" + } + } + }, + { + "id": 7, + "title": "GPU Thermal Profile (\u00b0C)", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "{{Hostname}} (\u00b0C)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10 + }, + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 65 + }, + { + "color": "red", + "value": 80 + } + ] + } + } + } + }, + { + "id": 8, + "title": "GPU Power Draw (Watts)", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE", + "legendFormat": "{{Hostname}} (Watts)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15 + }, + "unit": "watt" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 300, + "title": "\ud83e\udde0 vLLM Real-Time Inference Dynamics & Token Velocity", + "type": "row" + }, + { + "id": 9, + "title": "Token Generation Velocity (Tokens / Sec)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", + "legendFormat": "{{model_name}} (Out)", + "refId": "A" + }, + { + "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", + "legendFormat": "{{model_name}} (In)", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 25 + }, + "unit": "cps" + } + } + }, + { + "id": 10, + "title": "Inference Concurrency & Request Backlog", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "vllm:num_requests_running", + "legendFormat": "{{model_name}} Running", + "refId": "A" + }, + { + "expr": "vllm:num_requests_waiting", + "legendFormat": "{{model_name}} Queued", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 15 + }, + "unit": "short" + } + } + }, + { + "id": 11, + "title": "GPU KV Cache Memory Saturation", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 30 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "vllm:gpu_cache_usage_factor * 100", + "legendFormat": "{{model_name}} Cache Usage", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 30 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + } + }, + { + "id": 12, + "title": "NVIDIA Triton & Qdrant Operations", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "rate(nv_inference_request_success[2m])", + "legendFormat": "Triton Success/s", + "refId": "A" + }, + { + "expr": "cpu_cores_used", + "legendFormat": "Qdrant CPU Cores", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth" + }, + "unit": "short" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "ai", + "vllm", + "dcgm", + "gpu", + "snarfnet" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "SnarfNet AI Stack & Inference Intelligence", + "uid": "snarfnet-ai-stack", + "version": 1 +} \ No newline at end of file diff --git a/manifests/aerodrome/grafana/grafana.yaml b/manifests/aerodrome/grafana/grafana.yaml index 990c54d..d2fab0e 100644 --- a/manifests/aerodrome/grafana/grafana.yaml +++ b/manifests/aerodrome/grafana/grafana.yaml @@ -111,6 +111,15 @@ spec: subPath: grafana.ini - mountPath: /var/lib/grafana name: storage + - mountPath: /etc/grafana/provisioning/datasources/datasources.yaml + name: datasources-config + subPath: datasources.yaml + - mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml + name: dashboards-provider-config + subPath: dashboards.yaml + - mountPath: /var/lib/grafana-dashboards/ai-stack-dashboard.json + name: ai-stack-dashboard + subPath: ai-stack-dashboard.json dnsPolicy: ClusterFirst enableServiceLinks: true initContainers: @@ -154,6 +163,18 @@ spec: - name: storage persistentVolumeClaim: claimName: grafana + - configMap: + defaultMode: 420 + name: grafana-datasources + name: datasources-config + - configMap: + defaultMode: 420 + name: grafana-dashboards-provider + name: dashboards-provider-config + - configMap: + defaultMode: 420 + name: grafana-dashboard-ai-stack + name: ai-stack-dashboard --- apiVersion: v1 kind: Service @@ -250,3 +271,575 @@ spec: - hosts: - grafana.snarfnet.net secretName: firefly-tls-cert +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: default + labels: + app.kubernetes.io/name: grafana +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-provider + namespace: default + labels: + app.kubernetes.io/name: grafana +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: 'ai-stack-provider' + orgId: 1 + folder: 'AI Stack & Inference' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana-dashboards +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-ai-stack + namespace: default + labels: + app.kubernetes.io/name: grafana +data: + ai-stack-dashboard.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "\u26a1 AI Stack Executive Overview & Cloud Cost Offsets", + "type": "row" + }, + { + "id": 1, + "title": "Prompt Tokens Processed (In)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(vllm:prompt_tokens_total)", + "legendFormat": "Prompt Tokens", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 2, + "title": "Generation Tokens Synthesized (Out)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(vllm:generation_tokens_total)", + "legendFormat": "Gen Tokens", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + } + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center" + } + }, + { + "id": 3, + "title": "Homelab Cloud API Savings ($USD)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", + "legendFormat": "Net Savings", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + } + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + } + }, + { + "id": 4, + "title": "Active GPU Cluster Health", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "count(DCGM_FI_DEV_GPU_TEMP)", + "legendFormat": "Tesla T4s Active", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + } + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "\ud83c\udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)", + "type": "row" + }, + { + "id": 5, + "title": "GPU Core Utilization (%)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL", + "legendFormat": "{{Hostname}} ({{modelName}})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + } + }, + { + "id": 6, + "title": "VRAM Memory Allocated & Used", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", + "legendFormat": "{{Hostname}} VRAM Used", + "refId": "A" + }, + { + "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", + "legendFormat": "{{Hostname}} VRAM Total", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15 + }, + "unit": "bytes" + } + } + }, + { + "id": 7, + "title": "GPU Thermal Profile (\u00b0C)", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "{{Hostname}} (\u00b0C)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10 + }, + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 65 + }, + { + "color": "red", + "value": 80 + } + ] + } + } + } + }, + { + "id": 8, + "title": "GPU Power Draw (Watts)", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE", + "legendFormat": "{{Hostname}} (Watts)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15 + }, + "unit": "watt" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 300, + "title": "\ud83e\udde0 vLLM Real-Time Inference Dynamics & Token Velocity", + "type": "row" + }, + { + "id": 9, + "title": "Token Generation Velocity (Tokens / Sec)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", + "legendFormat": "{{model_name}} (Out)", + "refId": "A" + }, + { + "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", + "legendFormat": "{{model_name}} (In)", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 25 + }, + "unit": "cps" + } + } + }, + { + "id": 10, + "title": "Inference Concurrency & Request Backlog", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "vllm:num_requests_running", + "legendFormat": "{{model_name}} Running", + "refId": "A" + }, + { + "expr": "vllm:num_requests_waiting", + "legendFormat": "{{model_name}} Queued", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 15 + }, + "unit": "short" + } + } + }, + { + "id": 11, + "title": "GPU KV Cache Memory Saturation", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 30 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "vllm:gpu_cache_usage_factor * 100", + "legendFormat": "{{model_name}} Cache Usage", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 30 + }, + "unit": "percent", + "min": 0, + "max": 100 + } + } + }, + { + "id": 12, + "title": "NVIDIA Triton & Qdrant Operations", + "type": "timeseries", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 + }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "rate(nv_inference_request_success[2m])", + "legendFormat": "Triton Success/s", + "refId": "A" + }, + { + "expr": "cpu_cores_used", + "legendFormat": "Qdrant CPU Cores", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth" + }, + "unit": "short" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "ai", + "vllm", + "dcgm", + "gpu", + "snarfnet" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "SnarfNet AI Stack & Inference Intelligence", + "uid": "snarfnet-ai-stack", + "version": 1 + } diff --git a/manifests/aerodrome/prometheus/prometheus.yaml b/manifests/aerodrome/prometheus/prometheus.yaml new file mode 100644 index 0000000..fea60e5 --- /dev/null +++ b/manifests/aerodrome/prometheus/prometheus.yaml @@ -0,0 +1,182 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-storage + namespace: default + labels: + app.kubernetes.io/name: prometheus +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 50Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: default + labels: + app.kubernetes.io/name: prometheus +data: + prometheus.yml: | + global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'vllm-qwen' + metrics_path: /metrics + static_configs: + - targets: ['140.44.4.84:30800'] + labels: + model: 'Qwen2.5-7B-Instruct' + node: 'snaisvr44hrvstr04' + role: 'general-instruction' + + - job_name: 'vllm-gemma4' + metrics_path: /metrics + static_configs: + - targets: ['140.44.4.82:30804'] + labels: + model: 'Gemma-4-12B-it' + node: 'snaisvr44hrvstr02' + role: 'coding-agent' + + - job_name: 'vllm-deepseek-r1' + metrics_path: /metrics + static_configs: + - targets: ['140.44.4.81:30805'] + labels: + model: 'DeepSeek-R1-Distill-Qwen-14B' + node: 'snaisvr44hrvstr01' + role: 'reasoning-planner' + + - job_name: 'triton-inference-server' + metrics_path: /metrics + static_configs: + - targets: ['140.44.4.83:30803'] + labels: + app: 'triton-inference-server' + node: 'snaisvr44hrvstr03' + model: 'decision_engine' + + - job_name: 'qdrant' + metrics_path: /metrics + static_configs: + - targets: ['140.44.4.82:30633'] + labels: + app: 'qdrant' + node: 'snaisvr44hrvstr02' + + - job_name: 'nvidia-dcgm' + metrics_path: /metrics + static_configs: + - targets: ['140.44.4.81:9400'] + labels: + node: 'snaisvr44hrvstr01' + gpu_model: 'Tesla T4' + - targets: ['140.44.4.82:9400'] + labels: + node: 'snaisvr44hrvstr02' + gpu_model: 'Tesla T4' + - targets: ['140.44.4.83:9400'] + labels: + node: 'snaisvr44hrvstr03' + gpu_model: 'Tesla T4' + - targets: ['140.44.4.84:9400'] + labels: + node: 'snaisvr44hrvstr04' + gpu_model: 'Tesla T4' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: default + labels: + app.kubernetes.io/name: prometheus +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: prometheus + template: + metadata: + labels: + app.kubernetes.io/name: prometheus + spec: + securityContext: + fsGroup: 65534 + runAsUser: 65534 + runAsNonRoot: true + containers: + - name: prometheus + image: prom/prometheus:v2.54.1 + imagePullPolicy: IfNotPresent + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=90d" + - "--web.enable-lifecycle" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + ports: + - name: web + containerPort: 9090 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "4Gi" + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 30 + periodSeconds: 15 + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: storage + mountPath: /prometheus + volumes: + - name: config + configMap: + name: prometheus-config + - name: storage + persistentVolumeClaim: + claimName: prometheus-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: default + labels: + app.kubernetes.io/name: prometheus +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: prometheus + ports: + - name: web + port: 9090 + targetPort: 9090 diff --git a/manifests/spacedock/ai-stack/ai-stack.yaml b/manifests/spacedock/ai-stack/ai-stack.yaml index d0a580a..5a415ed 100644 --- a/manifests/spacedock/ai-stack/ai-stack.yaml +++ b/manifests/spacedock/ai-stack/ai-stack.yaml @@ -609,3 +609,53 @@ spec: port: 8002 targetPort: 8002 nodePort: 30803 + +--- +# ============================================================================== +# 4. NVIDIA DCGM EXPORTER (Tesla T4 Telemetry: Utilization, VRAM, Power, Temp) +# ============================================================================== +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dcgm-exporter + namespace: ai-stack + labels: + app.kubernetes.io/name: dcgm-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + template: + metadata: + labels: + app.kubernetes.io/name: dcgm-exporter + spec: + hostNetwork: true + containers: + - name: dcgm-exporter + image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04 + env: + - name: DCGM_EXPORTER_LISTEN + value: ":9400" + - name: DCGM_EXPORTER_KUBERNETES + value: "false" + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64" + securityContext: + privileged: true + ports: + - name: metrics + containerPort: 9400 + volumeMounts: + - name: dev + mountPath: /dev + - name: nvidia-lib + mountPath: /usr/local/nvidia/lib64 + readOnly: true + volumes: + - name: dev + hostPath: + path: /dev + - name: nvidia-lib + hostPath: + path: /usr/local/nvidia/lib64