apiVersion: apps/v1 kind: Deployment metadata: annotations: meta.helm.sh/release-name: grafana meta.helm.sh/release-namespace: default labels: app.kubernetes.io/instance: grafana app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 11.3.1 helm.sh/chart: grafana-8.6.4 name: grafana namespace: default spec: progressDeadlineSeconds: 600 replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: app.kubernetes.io/instance: grafana app.kubernetes.io/name: grafana strategy: type: Recreate template: metadata: annotations: cattle.io/timestamp: '2025-10-04T04:05:51Z' checksum/config: f6e2070ec7fff0d4bb82d9a8fabedb9934e42e64a00d72e25402d46520734840 checksum/sc-dashboard-provider-config: e70bf6a851099d385178a76de9757bb0bef8299da6d8443602590e44f05fdf24 checksum/secret: 4eccb44a21c8117c1dbfbfe43f2ac844b240cd2482bc9fba8e4047442d121dc2 kubectl.kubernetes.io/default-container: grafana kubectl.kubernetes.io/restartedAt: '2026-10-28T17:14:50.583825+00:00' labels: app.kubernetes.io/instance: grafana app.kubernetes.io/name: grafana app.kubernetes.io/version: 11.3.1 helm.sh/chart: grafana-8.6.4 spec: automountServiceAccountToken: true containers: - env: - name: POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: GF_SECURITY_ADMIN_USER valueFrom: secretKeyRef: key: admin-user name: grafana - name: GF_SECURITY_ADMIN_PASSWORD valueFrom: secretKeyRef: key: admin-password name: grafana - name: GF_PATHS_DATA value: /var/lib/grafana/ - name: GF_PATHS_LOGS value: /var/log/grafana - name: GF_PATHS_PLUGINS value: /var/lib/grafana/plugins - name: GF_PATHS_PROVISIONING value: /etc/grafana/provisioning image: docker.io/grafana/grafana:11.6.16 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 10 httpGet: path: /api/health port: 3000 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 name: grafana ports: - containerPort: 3000 name: grafana protocol: TCP - containerPort: 9094 name: gossip-tcp protocol: TCP - containerPort: 9094 name: gossip-udp protocol: UDP readinessProbe: failureThreshold: 3 httpGet: path: /api/health port: 3000 scheme: HTTP periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 resources: {} securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL seccompProfile: type: RuntimeDefault terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /etc/grafana/grafana.ini name: config subPath: grafana.ini - mountPath: /var/lib/grafana name: storage - mountPath: /etc/grafana/provisioning/datasources/datasources.yaml name: datasources-config subPath: datasources.yaml - mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml name: dashboards-provider-config subPath: dashboards.yaml - mountPath: /var/lib/grafana-dashboards/ai-stack-dashboard.json name: ai-stack-dashboard subPath: ai-stack-dashboard.json dnsPolicy: ClusterFirst enableServiceLinks: true initContainers: - command: - chown - -R - 472:472 - /var/lib/grafana image: docker.io/library/busybox:1.31.1 imagePullPolicy: IfNotPresent name: init-chown-data resources: {} securityContext: capabilities: add: - CHOWN runAsNonRoot: false runAsUser: 0 seccompProfile: type: RuntimeDefault terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/lib/grafana name: storage restartPolicy: Always schedulerName: default-scheduler securityContext: fsGroup: 472 runAsGroup: 472 runAsNonRoot: true runAsUser: 472 serviceAccount: grafana serviceAccountName: grafana terminationGracePeriodSeconds: 30 volumes: - configMap: defaultMode: 420 name: grafana name: config - name: storage persistentVolumeClaim: claimName: grafana - configMap: defaultMode: 420 name: grafana-datasources name: datasources-config - configMap: defaultMode: 420 name: grafana-dashboards-provider name: dashboards-provider-config - configMap: defaultMode: 420 name: grafana-dashboard-ai-stack name: ai-stack-dashboard --- apiVersion: v1 kind: Service metadata: annotations: meta.helm.sh/release-name: grafana meta.helm.sh/release-namespace: default labels: app.kubernetes.io/instance: grafana app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 11.3.1 helm.sh/chart: grafana-8.6.4 name: grafana namespace: default spec: clusterIP: 10.43.216.183 clusterIPs: - 10.43.216.183 internalTrafficPolicy: Cluster ipFamilies: - IPv4 ipFamilyPolicy: SingleStack ports: - name: service port: 8080 protocol: TCP targetPort: 3000 selector: app.kubernetes.io/instance: grafana app.kubernetes.io/name: grafana sessionAffinity: None type: ClusterIP --- apiVersion: v1 kind: Service metadata: annotations: metallb.io/ip-allocated-from-pool: lan-pool metallb.universe.tf/ip-allocated-from-pool: default-pool metallb.universe.tf/loadBalancerIP: 140.44.4.32 finalizers: - service.kubernetes.io/load-balancer-cleanup labels: app.kubernetes.io/instance: grafana app.kubernetes.io/name: grafana name: grafana-lb namespace: default spec: allocateLoadBalancerNodePorts: true clusterIP: 10.43.240.82 clusterIPs: - 10.43.240.82 externalTrafficPolicy: Cluster internalTrafficPolicy: Cluster ipFamilies: - IPv4 ipFamilyPolicy: SingleStack loadBalancerIP: 140.44.4.32 ports: - name: grafana-lb nodePort: 32758 port: 88 protocol: TCP targetPort: 3000 selector: app.kubernetes.io/instance: grafana app.kubernetes.io/name: grafana sessionAffinity: None type: LoadBalancer --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: labels: app.kubernetes.io/instance: grafana app.kubernetes.io/name: grafana name: grafana namespace: default spec: ingressClassName: traefik rules: - host: grafana.snarfnet.net http: paths: - backend: service: name: grafana-lb port: number: 80 path: / pathType: ImplementationSpecific tls: - hosts: - grafana.snarfnet.net secretName: firefly-tls-cert --- apiVersion: v1 kind: ConfigMap metadata: name: grafana-datasources namespace: default labels: app.kubernetes.io/name: grafana data: datasources.yaml: | apiVersion: 1 datasources: - name: Prometheus type: prometheus access: proxy url: http://prometheus:9090 isDefault: true jsonData: timeInterval: 15s --- apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboards-provider namespace: default labels: app.kubernetes.io/name: grafana data: dashboards.yaml: | apiVersion: 1 providers: - name: 'ai-stack-provider' orgId: 1 folder: 'AI Stack & Inference' type: file disableDeletion: false editable: true options: path: /var/lib/grafana-dashboards --- apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-ai-stack namespace: default labels: app.kubernetes.io/name: grafana data: ai-stack-dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "title": "\u26a1 AI Stack Executive Overview & Cloud Cost Offsets", "type": "row" }, { "id": 1, "title": "Prompt Tokens Processed (In)", "type": "stat", "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "datasource": "Prometheus", "targets": [ { "expr": "sum(vllm:prompt_tokens_total)", "legendFormat": "Prompt Tokens", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "locale" } }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" } }, { "id": 2, "title": "Generation Tokens Synthesized (Out)", "type": "stat", "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, "datasource": "Prometheus", "targets": [ { "expr": "sum(vllm:generation_tokens_total)", "legendFormat": "Gen Tokens", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "locale" } }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" } }, { "id": 3, "title": "Homelab Cloud API Savings ($USD)", "type": "stat", "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, "datasource": "Prometheus", "targets": [ { "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", "legendFormat": "Net Savings", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "currencyUSD" } }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center" } }, { "id": 4, "title": "Active GPU Cluster Health", "type": "stat", "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "datasource": "Prometheus", "targets": [ { "expr": "count(DCGM_FI_DEV_GPU_TEMP)", "legendFormat": "Tesla T4s Active", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] }, "unit": "short" } }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center" } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 200, "title": "\ud83c\udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)", "type": "row" }, { "id": 5, "title": "GPU Core Utilization (%)", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "datasource": "Prometheus", "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "{{Hostname}} ({{modelName}})", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20 }, "unit": "percent", "min": 0, "max": 100 } } }, { "id": 6, "title": "VRAM Memory Allocated & Used", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "datasource": "Prometheus", "targets": [ { "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Used", "refId": "A" }, { "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Total", "refId": "B" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 }, "unit": "bytes" } } }, { "id": 7, "title": "GPU Thermal Profile (\u00b0C)", "type": "timeseries", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "datasource": "Prometheus", "targets": [ { "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "{{Hostname}} (\u00b0C)", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 }, "unit": "celsius", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 65 }, { "color": "red", "value": 80 } ] } } } }, { "id": 8, "title": "GPU Power Draw (Watts)", "type": "timeseries", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "datasource": "Prometheus", "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "{{Hostname}} (Watts)", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 }, "unit": "watt" } } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 300, "title": "\ud83e\udde0 vLLM Real-Time Inference Dynamics & Token Velocity", "type": "row" }, { "id": 9, "title": "Token Generation Velocity (Tokens / Sec)", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "datasource": "Prometheus", "targets": [ { "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (Out)", "refId": "A" }, { "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (In)", "refId": "B" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 25 }, "unit": "cps" } } }, { "id": 10, "title": "Inference Concurrency & Request Backlog", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "datasource": "Prometheus", "targets": [ { "expr": "vllm:num_requests_running", "legendFormat": "{{model_name}} Running", "refId": "A" }, { "expr": "vllm:num_requests_waiting", "legendFormat": "{{model_name}} Queued", "refId": "B" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "stepAfter", "fillOpacity": 15 }, "unit": "short" } } }, { "id": 11, "title": "GPU KV Cache Memory Saturation", "type": "timeseries", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 }, "datasource": "Prometheus", "targets": [ { "expr": "vllm:gpu_cache_usage_factor * 100", "legendFormat": "{{model_name}} Cache Usage", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 30 }, "unit": "percent", "min": 0, "max": 100 } } }, { "id": 12, "title": "NVIDIA Triton & Qdrant Operations", "type": "timeseries", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 }, "datasource": "Prometheus", "targets": [ { "expr": "rate(nv_inference_request_success[2m])", "legendFormat": "Triton Success/s", "refId": "A" }, { "expr": "cpu_cores_used", "legendFormat": "Qdrant CPU Cores", "refId": "B" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth" }, "unit": "short" } } } ], "refresh": "10s", "schemaVersion": 39, "tags": [ "ai", "vllm", "dcgm", "gpu", "snarfnet" ], "templating": { "list": [] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m" ] }, "timezone": "browser", "title": "SnarfNet AI Stack & Inference Intelligence", "uid": "snarfnet-ai-stack", "version": 1 }