apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-storage
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn
  resources:
    requests:
      storage: 50Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      scrape_timeout: 10s
      evaluation_interval: 15s

    scrape_configs:
      - job_name: 'prometheus'
        static_configs:
          - targets: ['localhost:9090']

      - job_name: 'vllm-qwen'
        metrics_path: /metrics
        static_configs:
          - targets: ['140.44.4.84:30800']
            labels:
              model: 'Qwen2.5-7B-Instruct'
              node: 'snaisvr44hrvstr04'
              role: 'general-instruction'

      - job_name: 'vllm-gemma4'
        metrics_path: /metrics
        static_configs:
          - targets: ['140.44.4.82:30804']
            labels:
              model: 'Gemma-4-12B-it'
              node: 'snaisvr44hrvstr02'
              role: 'coding-agent'

      - job_name: 'vllm-deepseek-r1'
        metrics_path: /metrics
        static_configs:
          - targets: ['140.44.4.81:30805']
            labels:
              model: 'DeepSeek-R1-Distill-Qwen-14B'
              node: 'snaisvr44hrvstr01'
              role: 'reasoning-planner'

      - job_name: 'triton-inference-server'
        metrics_path: /metrics
        static_configs:
          - targets: ['140.44.4.83:30803']
            labels:
              app: 'triton-inference-server'
              node: 'snaisvr44hrvstr03'
              model: 'decision_engine'

      - job_name: 'qdrant'
        metrics_path: /metrics
        static_configs:
          - targets: ['140.44.4.82:30633']
            labels:
              app: 'qdrant'
              node: 'snaisvr44hrvstr02'

      - job_name: 'nvidia-dcgm'
        metrics_path: /metrics
        static_configs:
          - targets: ['140.44.4.81:9400']
            labels:
              node: 'snaisvr44hrvstr01'
              gpu_model: 'Tesla T4'
          - targets: ['140.44.4.82:9400']
            labels:
              node: 'snaisvr44hrvstr02'
              gpu_model: 'Tesla T4'
          - targets: ['140.44.4.83:9400']
            labels:
              node: 'snaisvr44hrvstr03'
              gpu_model: 'Tesla T4'
          - targets: ['140.44.4.84:9400']
            labels:
              node: 'snaisvr44hrvstr04'
              gpu_model: 'Tesla T4'
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus
  template:
    metadata:
      labels:
        app.kubernetes.io/name: prometheus
    spec:
      securityContext:
        fsGroup: 65534
        runAsUser: 65534
        runAsNonRoot: true
      containers:
        - name: prometheus
          image: prom/prometheus:v2.54.1
          imagePullPolicy: IfNotPresent
          args:
            - "--config.file=/etc/prometheus/prometheus.yml"
            - "--storage.tsdb.path=/prometheus"
            - "--storage.tsdb.retention.time=90d"
            - "--web.enable-lifecycle"
            - "--web.console.libraries=/usr/share/prometheus/console_libraries"
            - "--web.console.templates=/usr/share/prometheus/consoles"
          ports:
            - name: web
              containerPort: 9090
          resources:
            requests:
              cpu: "500m"
              memory: "1Gi"
            limits:
              cpu: "2"
              memory: "4Gi"
          readinessProbe:
            httpGet:
              path: /-/ready
              port: 9090
            initialDelaySeconds: 10
            periodSeconds: 5
          livenessProbe:
            httpGet:
              path: /-/healthy
              port: 9090
            initialDelaySeconds: 30
            periodSeconds: 15
          volumeMounts:
            - name: config
              mountPath: /etc/prometheus
            - name: storage
              mountPath: /prometheus
      volumes:
        - name: config
          configMap:
            name: prometheus-config
        - name: storage
          persistentVolumeClaim:
            claimName: prometheus-storage
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: prometheus
  ports:
    - name: web
      port: 9090
      targetPort: 9090