Snarf/homelab-gitops
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): enable node-exporter and kubernetes-cadvisor scraping in prometheus

Browse Source
This commit is contained in:
Snarf
2026-11-12 13:06:18 -05:00
parent 28f199f604
commit dfa2578723
2 changed files with 214 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
argocd/aerodrome/prometheus-app.yaml
View File

@@ -25,3 +25,4 @@ spec:
syncOptions:
- CreateNamespace=true
- ServerSideApply=true

327
manifests/aerodrome/prometheus/prometheus.yaml
View File

@@ -1,4 +1,53 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: default
labels:
app.kubernetes.io/name: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
rules:
- apiGroups:
- ''
resources:
- nodes
- nodes/proxy
- nodes/metrics
- services
- endpoints
- pods
verbs:
- get
- list
- watch
- nonResourceURLs:
- /metrics
- /metrics/cadvisor
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: default
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-storage
@@ -7,7 +56,7 @@ metadata:
app.kubernetes.io/name: prometheus
spec:
accessModes:
- ReadWriteOnce
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
@@ -26,75 +75,124 @@ data:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'vllm-qwen'
metrics_path: /metrics
static_configs:
- targets: ['140.44.4.84:30800']
labels:
model: 'Qwen2.5-7B-Instruct'
node: 'snaisvr44hrvstr04'
role: 'general-instruction'
- job_name: 'vllm-gemma4'
metrics_path: /metrics
static_configs:
- targets: ['140.44.4.82:30804']
labels:
model: 'Gemma-4-12B-it'
node: 'snaisvr44hrvstr02'
role: 'coding-agent'
- job_name: 'vllm-deepseek-r1'
metrics_path: /metrics
static_configs:
- targets: ['140.44.4.81:30805']
labels:
model: 'DeepSeek-R1-Distill-Qwen-14B'
node: 'snaisvr44hrvstr01'
role: 'reasoning-planner'
- job_name: 'triton-inference-server'
metrics_path: /metrics
static_configs:
- targets: ['140.44.4.83:30803']
labels:
app: 'triton-inference-server'
node: 'snaisvr44hrvstr03'
model: 'decision_engine'
- job_name: 'qdrant'
metrics_path: /metrics
static_configs:
- targets: ['140.44.4.82:30633']
labels:
app: 'qdrant'
node: 'snaisvr44hrvstr02'
- job_name: 'nvidia-dcgm'
metrics_path: /metrics
static_configs:
- targets: ['140.44.4.81:9400']
labels:
node: 'snaisvr44hrvstr01'
gpu_model: 'Tesla T4'
- targets: ['140.44.4.82:9400']
labels:
node: 'snaisvr44hrvstr02'
gpu_model: 'Tesla T4'
- targets: ['140.44.4.83:9400']
labels:
node: 'snaisvr44hrvstr03'
gpu_model: 'Tesla T4'
- targets: ['140.44.4.84:9400']
labels:
node: 'snaisvr44hrvstr04'
gpu_model: 'Tesla T4'
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- job_name: vllm-qwen
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.84:30800
labels:
model: Qwen2.5-7B-Instruct
node: snaisvr44hrvstr04
role: general-instruction
- job_name: vllm-gemma4
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.82:30804
labels:
model: Gemma-4-12B-it
node: snaisvr44hrvstr02
role: coding-agent
- job_name: vllm-deepseek-r1
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.81:30805
labels:
model: DeepSeek-R1-Distill-Qwen-14B
node: snaisvr44hrvstr01
role: reasoning-planner
- job_name: triton-inference-server
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.83:30803
labels:
app: triton-inference-server
node: snaisvr44hrvstr03
model: decision_engine
- job_name: qdrant
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.82:30633
labels:
app: qdrant
node: snaisvr44hrvstr02
- job_name: nvidia-dcgm
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.81:9400
labels:
node: snaisvr44hrvstr01
gpu_model: Tesla T4
- targets:
- 140.44.4.82:9400
labels:
node: snaisvr44hrvstr02
gpu_model: Tesla T4
- targets:
- 140.44.4.83:9400
labels:
node: snaisvr44hrvstr03
gpu_model: Tesla T4
- targets:
- 140.44.4.84:9400
labels:
node: snaisvr44hrvstr04
gpu_model: Tesla T4
- job_name: node-exporter
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.11:9100
- 140.44.4.12:9100
- 140.44.4.13:9100
- 140.44.4.21:9100
- 140.44.4.22:9100
- 140.44.4.23:9100
- 140.44.4.24:9100
- 140.44.4.25:9100
- 140.44.4.26:9100
- 140.44.4.27:9100
labels:
cluster: aerodrome
- targets:
- 140.44.4.41:9100
- 140.44.4.42:9100
- 140.44.4.43:9100
- 140.44.4.51:9100
- 140.44.4.52:9100
- 140.44.4.53:9100
- 140.44.4.54:9100
- 140.44.4.55:9100
- 140.44.4.56:9100
- 140.44.4.57:9100
labels:
cluster: spaceport
- job_name: kubernetes-cadvisor
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels:
- __meta_kubernetes_node_name
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
---
apiVersion: apps/v1
kind: Deployment
@@ -120,50 +218,51 @@ spec:
runAsUser: 65534
runAsNonRoot: true
containers:
- name: prometheus
image: prom/prometheus:v2.54.1
imagePullPolicy: IfNotPresent
args:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=90d"
- "--web.enable-lifecycle"
- "--web.console.libraries=/usr/share/prometheus/console_libraries"
- "--web.console.templates=/usr/share/prometheus/consoles"
ports:
- name: web
containerPort: 9090
resources:
requests:
cpu: "500m"
memory: "1Gi"
limits:
cpu: "2"
memory: "4Gi"
readinessProbe:
httpGet:
path: /-/ready
port: 9090
initialDelaySeconds: 10
periodSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: 9090
initialDelaySeconds: 30
periodSeconds: 15
volumeMounts:
- name: config
mountPath: /etc/prometheus
- name: storage
mountPath: /prometheus
volumes:
- name: prometheus
image: prom/prometheus:v2.54.1
imagePullPolicy: IfNotPresent
args:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=90d
- --web.enable-lifecycle
- --web.console.libraries=/usr/share/prometheus/console_libraries
- --web.console.templates=/usr/share/prometheus/consoles
ports:
- name: web
containerPort: 9090
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: '2'
memory: 4Gi
readinessProbe:
httpGet:
path: /-/ready
port: 9090
initialDelaySeconds: 10
periodSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: 9090
initialDelaySeconds: 30
periodSeconds: 15
volumeMounts:
- name: config
configMap:
name: prometheus-config
mountPath: /etc/prometheus
- name: storage
persistentVolumeClaim:
claimName: prometheus-storage
mountPath: /prometheus
volumes:
- name: config
configMap:
name: prometheus-config
- name: storage
persistentVolumeClaim:
claimName: prometheus-storage
serviceAccountName: prometheus
---
apiVersion: v1
kind: Service
@@ -177,6 +276,6 @@ spec:
selector:
app.kubernetes.io/name: prometheus
ports:
- name: web
port: 9090
targetPort: 9090
- name: web
port: 9090
targetPort: 9090