apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
  labels:
    app.kubernetes.io/name: prometheus
rules:
- apiGroups:
  - ''
  resources:
  - nodes
  - nodes/proxy
  - nodes/metrics
  - services
  - endpoints
  - pods
  verbs:
  - get
  - list
  - watch
- nonResourceURLs:
  - /metrics
  - /metrics/cadvisor
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
  labels:
    app.kubernetes.io/name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: default
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-storage
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
spec:
  accessModes:
  - ReadWriteOnce
  storageClassName: longhorn
  resources:
    requests:
      storage: 50Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      scrape_timeout: 10s
      evaluation_interval: 15s
    scrape_configs:
    - job_name: prometheus
      static_configs:
      - targets:
        - localhost:9090
    - job_name: vllm-qwen
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.84:30800
        labels:
          model: Qwen2.5-7B-Instruct
          node: snaisvr44hrvstr04
          role: general-instruction
    - job_name: vllm-gemma4
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.82:30804
        labels:
          model: Gemma-4-12B-it
          node: snaisvr44hrvstr02
          role: coding-agent
    - job_name: vllm-deepseek-r1
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.81:30805
        labels:
          model: DeepSeek-R1-Distill-Qwen-14B
          node: snaisvr44hrvstr01
          role: reasoning-planner
    - job_name: triton-inference-server
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.83:30803
        labels:
          app: triton-inference-server
          node: snaisvr44hrvstr03
          model: decision_engine
    - job_name: qdrant
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.82:30633
        labels:
          app: qdrant
          node: snaisvr44hrvstr02
    - job_name: nvidia-dcgm
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.81:9400
        labels:
          node: snaisvr44hrvstr01
          gpu_model: Tesla T4
      - targets:
        - 140.44.4.82:9400
        labels:
          node: snaisvr44hrvstr02
          gpu_model: Tesla T4
      - targets:
        - 140.44.4.83:9400
        labels:
          node: snaisvr44hrvstr03
          gpu_model: Tesla T4
      - targets:
        - 140.44.4.84:9400
        labels:
          node: snaisvr44hrvstr04
          gpu_model: Tesla T4
    - job_name: node-exporter
      metrics_path: /metrics
      static_configs:
      - targets:
        - 140.44.4.11:9100
        - 140.44.4.12:9100
        - 140.44.4.13:9100
        - 140.44.4.21:9100
        - 140.44.4.22:9100
        - 140.44.4.23:9100
        - 140.44.4.24:9100
        - 140.44.4.25:9100
        - 140.44.4.26:9100
        - 140.44.4.27:9100
        labels:
          cluster: aerodrome
      - targets:
        - 140.44.4.41:9100
        - 140.44.4.42:9100
        - 140.44.4.43:9100
        - 140.44.4.51:9100
        - 140.44.4.52:9100
        - 140.44.4.53:9100
        - 140.44.4.54:9100
        - 140.44.4.55:9100
        - 140.44.4.56:9100
        - 140.44.4.57:9100
        labels:
          cluster: spaceport
    - job_name: kubernetes-cadvisor
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels:
        - __meta_kubernetes_node_name
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus
  template:
    metadata:
      labels:
        app.kubernetes.io/name: prometheus
    spec:
      securityContext:
        fsGroup: 65534
        runAsUser: 65534
        runAsNonRoot: true
      containers:
      - name: prometheus
        image: prom/prometheus:v2.54.1
        imagePullPolicy: IfNotPresent
        args:
        - --config.file=/etc/prometheus/prometheus.yml
        - --storage.tsdb.path=/prometheus
        - --storage.tsdb.retention.time=90d
        - --web.enable-lifecycle
        - --web.console.libraries=/usr/share/prometheus/console_libraries
        - --web.console.templates=/usr/share/prometheus/consoles
        ports:
        - name: web
          containerPort: 9090
        resources:
          requests:
            cpu: 500m
            memory: 1Gi
          limits:
            cpu: '2'
            memory: 4Gi
        readinessProbe:
          httpGet:
            path: /-/ready
            port: 9090
          initialDelaySeconds: 10
          periodSeconds: 5
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: 9090
          initialDelaySeconds: 30
          periodSeconds: 15
        volumeMounts:
        - name: config
          mountPath: /etc/prometheus
        - name: storage
          mountPath: /prometheus
      volumes:
      - name: config
        configMap:
          name: prometheus-config
      - name: storage
        persistentVolumeClaim:
          claimName: prometheus-storage
      serviceAccountName: prometheus
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: default
  labels:
    app.kubernetes.io/name: prometheus
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: prometheus
  ports:
  - name: web
    port: 9090
    targetPort: 9090