diff --git a/argocd/aerodrome/prometheus-app.yaml b/argocd/aerodrome/prometheus-app.yaml index 40dd927..8bb294c 100644 --- a/argocd/aerodrome/prometheus-app.yaml +++ b/argocd/aerodrome/prometheus-app.yaml @@ -25,3 +25,4 @@ spec: syncOptions: - CreateNamespace=true - ServerSideApply=true + diff --git a/manifests/aerodrome/prometheus/prometheus.yaml b/manifests/aerodrome/prometheus/prometheus.yaml index fea60e5..e7bf565 100644 --- a/manifests/aerodrome/prometheus/prometheus.yaml +++ b/manifests/aerodrome/prometheus/prometheus.yaml @@ -1,4 +1,53 @@ apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: default + labels: + app.kubernetes.io/name: prometheus +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus + labels: + app.kubernetes.io/name: prometheus +rules: +- apiGroups: + - '' + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + verbs: + - get + - list + - watch +- nonResourceURLs: + - /metrics + - /metrics/cadvisor + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus + labels: + app.kubernetes.io/name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: default +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: prometheus-storage @@ -7,7 +56,7 @@ metadata: app.kubernetes.io/name: prometheus spec: accessModes: - - ReadWriteOnce + - ReadWriteOnce storageClassName: longhorn resources: requests: @@ -26,75 +75,124 @@ data: scrape_interval: 15s scrape_timeout: 10s evaluation_interval: 15s - scrape_configs: - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - - - job_name: 'vllm-qwen' - metrics_path: /metrics - static_configs: - - targets: ['140.44.4.84:30800'] - labels: - model: 'Qwen2.5-7B-Instruct' - node: 'snaisvr44hrvstr04' - role: 'general-instruction' - - - job_name: 'vllm-gemma4' - metrics_path: /metrics - static_configs: - - targets: ['140.44.4.82:30804'] - labels: - model: 'Gemma-4-12B-it' - node: 'snaisvr44hrvstr02' - role: 'coding-agent' - - - job_name: 'vllm-deepseek-r1' - metrics_path: /metrics - static_configs: - - targets: ['140.44.4.81:30805'] - labels: - model: 'DeepSeek-R1-Distill-Qwen-14B' - node: 'snaisvr44hrvstr01' - role: 'reasoning-planner' - - - job_name: 'triton-inference-server' - metrics_path: /metrics - static_configs: - - targets: ['140.44.4.83:30803'] - labels: - app: 'triton-inference-server' - node: 'snaisvr44hrvstr03' - model: 'decision_engine' - - - job_name: 'qdrant' - metrics_path: /metrics - static_configs: - - targets: ['140.44.4.82:30633'] - labels: - app: 'qdrant' - node: 'snaisvr44hrvstr02' - - - job_name: 'nvidia-dcgm' - metrics_path: /metrics - static_configs: - - targets: ['140.44.4.81:9400'] - labels: - node: 'snaisvr44hrvstr01' - gpu_model: 'Tesla T4' - - targets: ['140.44.4.82:9400'] - labels: - node: 'snaisvr44hrvstr02' - gpu_model: 'Tesla T4' - - targets: ['140.44.4.83:9400'] - labels: - node: 'snaisvr44hrvstr03' - gpu_model: 'Tesla T4' - - targets: ['140.44.4.84:9400'] - labels: - node: 'snaisvr44hrvstr04' - gpu_model: 'Tesla T4' + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - job_name: vllm-qwen + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.84:30800 + labels: + model: Qwen2.5-7B-Instruct + node: snaisvr44hrvstr04 + role: general-instruction + - job_name: vllm-gemma4 + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.82:30804 + labels: + model: Gemma-4-12B-it + node: snaisvr44hrvstr02 + role: coding-agent + - job_name: vllm-deepseek-r1 + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.81:30805 + labels: + model: DeepSeek-R1-Distill-Qwen-14B + node: snaisvr44hrvstr01 + role: reasoning-planner + - job_name: triton-inference-server + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.83:30803 + labels: + app: triton-inference-server + node: snaisvr44hrvstr03 + model: decision_engine + - job_name: qdrant + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.82:30633 + labels: + app: qdrant + node: snaisvr44hrvstr02 + - job_name: nvidia-dcgm + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.81:9400 + labels: + node: snaisvr44hrvstr01 + gpu_model: Tesla T4 + - targets: + - 140.44.4.82:9400 + labels: + node: snaisvr44hrvstr02 + gpu_model: Tesla T4 + - targets: + - 140.44.4.83:9400 + labels: + node: snaisvr44hrvstr03 + gpu_model: Tesla T4 + - targets: + - 140.44.4.84:9400 + labels: + node: snaisvr44hrvstr04 + gpu_model: Tesla T4 + - job_name: node-exporter + metrics_path: /metrics + static_configs: + - targets: + - 140.44.4.11:9100 + - 140.44.4.12:9100 + - 140.44.4.13:9100 + - 140.44.4.21:9100 + - 140.44.4.22:9100 + - 140.44.4.23:9100 + - 140.44.4.24:9100 + - 140.44.4.25:9100 + - 140.44.4.26:9100 + - 140.44.4.27:9100 + labels: + cluster: aerodrome + - targets: + - 140.44.4.41:9100 + - 140.44.4.42:9100 + - 140.44.4.43:9100 + - 140.44.4.51:9100 + - 140.44.4.52:9100 + - 140.44.4.53:9100 + - 140.44.4.54:9100 + - 140.44.4.55:9100 + - 140.44.4.56:9100 + - 140.44.4.57:9100 + labels: + cluster: spaceport + - job_name: kubernetes-cadvisor + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: + - __meta_kubernetes_node_name + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor --- apiVersion: apps/v1 kind: Deployment @@ -120,50 +218,51 @@ spec: runAsUser: 65534 runAsNonRoot: true containers: - - name: prometheus - image: prom/prometheus:v2.54.1 - imagePullPolicy: IfNotPresent - args: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus" - - "--storage.tsdb.retention.time=90d" - - "--web.enable-lifecycle" - - "--web.console.libraries=/usr/share/prometheus/console_libraries" - - "--web.console.templates=/usr/share/prometheus/consoles" - ports: - - name: web - containerPort: 9090 - resources: - requests: - cpu: "500m" - memory: "1Gi" - limits: - cpu: "2" - memory: "4Gi" - readinessProbe: - httpGet: - path: /-/ready - port: 9090 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /-/healthy - port: 9090 - initialDelaySeconds: 30 - periodSeconds: 15 - volumeMounts: - - name: config - mountPath: /etc/prometheus - - name: storage - mountPath: /prometheus - volumes: + - name: prometheus + image: prom/prometheus:v2.54.1 + imagePullPolicy: IfNotPresent + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=90d + - --web.enable-lifecycle + - --web.console.libraries=/usr/share/prometheus/console_libraries + - --web.console.templates=/usr/share/prometheus/consoles + ports: + - name: web + containerPort: 9090 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: '2' + memory: 4Gi + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 30 + periodSeconds: 15 + volumeMounts: - name: config - configMap: - name: prometheus-config + mountPath: /etc/prometheus - name: storage - persistentVolumeClaim: - claimName: prometheus-storage + mountPath: /prometheus + volumes: + - name: config + configMap: + name: prometheus-config + - name: storage + persistentVolumeClaim: + claimName: prometheus-storage + serviceAccountName: prometheus --- apiVersion: v1 kind: Service @@ -177,6 +276,6 @@ spec: selector: app.kubernetes.io/name: prometheus ports: - - name: web - port: 9090 - targetPort: 9090 + - name: web + port: 9090 + targetPort: 9090