feat(ai-stack): add DCGM exporter, Prometheus scraper, and Grafana AI Stack intelligence dashboard

manifests/aerodrome/prometheus/prometheus.yaml

@@ -0,0 +1,182 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheus-storage
+  namespace: default
+  labels:
+    app.kubernetes.io/name: prometheus
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: default
+  labels:
+    app.kubernetes.io/name: prometheus
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 15s
+      scrape_timeout: 10s
+      evaluation_interval: 15s
+
+    scrape_configs:
+      - job_name: 'prometheus'
+        static_configs:
+          - targets: ['localhost:9090']
+
+      - job_name: 'vllm-qwen'
+        metrics_path: /metrics
+        static_configs:
+          - targets: ['140.44.4.84:30800']
+            labels:
+              model: 'Qwen2.5-7B-Instruct'
+              node: 'snaisvr44hrvstr04'
+              role: 'general-instruction'
+
+      - job_name: 'vllm-gemma4'
+        metrics_path: /metrics
+        static_configs:
+          - targets: ['140.44.4.82:30804']
+            labels:
+              model: 'Gemma-4-12B-it'
+              node: 'snaisvr44hrvstr02'
+              role: 'coding-agent'
+
+      - job_name: 'vllm-deepseek-r1'
+        metrics_path: /metrics
+        static_configs:
+          - targets: ['140.44.4.81:30805']
+            labels:
+              model: 'DeepSeek-R1-Distill-Qwen-14B'
+              node: 'snaisvr44hrvstr01'
+              role: 'reasoning-planner'
+
+      - job_name: 'triton-inference-server'
+        metrics_path: /metrics
+        static_configs:
+          - targets: ['140.44.4.83:30803']
+            labels:
+              app: 'triton-inference-server'
+              node: 'snaisvr44hrvstr03'
+              model: 'decision_engine'
+
+      - job_name: 'qdrant'
+        metrics_path: /metrics
+        static_configs:
+          - targets: ['140.44.4.82:30633']
+            labels:
+              app: 'qdrant'
+              node: 'snaisvr44hrvstr02'
+
+      - job_name: 'nvidia-dcgm'
+        metrics_path: /metrics
+        static_configs:
+          - targets: ['140.44.4.81:9400']
+            labels:
+              node: 'snaisvr44hrvstr01'
+              gpu_model: 'Tesla T4'
+          - targets: ['140.44.4.82:9400']
+            labels:
+              node: 'snaisvr44hrvstr02'
+              gpu_model: 'Tesla T4'
+          - targets: ['140.44.4.83:9400']
+            labels:
+              node: 'snaisvr44hrvstr03'
+              gpu_model: 'Tesla T4'
+          - targets: ['140.44.4.84:9400']
+            labels:
+              node: 'snaisvr44hrvstr04'
+              gpu_model: 'Tesla T4'
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: default
+  labels:
+    app.kubernetes.io/name: prometheus
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: prometheus
+    spec:
+      securityContext:
+        fsGroup: 65534
+        runAsUser: 65534
+        runAsNonRoot: true
+      containers:
+        - name: prometheus
+          image: prom/prometheus:v2.54.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--storage.tsdb.path=/prometheus"
+            - "--storage.tsdb.retention.time=90d"
+            - "--web.enable-lifecycle"
+            - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+            - "--web.console.templates=/usr/share/prometheus/consoles"
+          ports:
+            - name: web
+              containerPort: 9090
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"
+            limits:
+              cpu: "2"
+              memory: "4Gi"
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: 9090
+            initialDelaySeconds: 30
+            periodSeconds: 15
+          volumeMounts:
+            - name: config
+              mountPath: /etc/prometheus
+            - name: storage
+              mountPath: /prometheus
+      volumes:
+        - name: config
+          configMap:
+            name: prometheus-config
+        - name: storage
+          persistentVolumeClaim:
+            claimName: prometheus-storage
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: default
+  labels:
+    app.kubernetes.io/name: prometheus
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: prometheus
+  ports:
+    - name: web
+      port: 9090
+      targetPort: 9090