Snarf/homelab-gitops
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): enable node-exporter and kubernetes-cadvisor scraping in prometheus

Browse Source
This commit is contained in:
Snarf
2026-11-12 13:06:18 -05:00
parent 28f199f604
commit dfa2578723
2 changed files with 214 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
argocd/aerodrome/prometheus-app.yaml
View File

@@ -25,3 +25,4 @@ spec:
syncOptions: syncOptions:
- CreateNamespace=true - CreateNamespace=true
- ServerSideApply=true - ServerSideApply=true

327
manifests/aerodrome/prometheus/prometheus.yaml
View File

@@ -1,4 +1,53 @@
apiVersion: v1 apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: default
labels:
app.kubernetes.io/name: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
rules:
- apiGroups:
- ''
resources:
- nodes
- nodes/proxy
- nodes/metrics
- services
- endpoints
- pods
verbs:
- get
- list
- watch
- nonResourceURLs:
- /metrics
- /metrics/cadvisor
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: default
---
apiVersion: v1
kind: PersistentVolumeClaim kind: PersistentVolumeClaim
metadata: metadata:
name: prometheus-storage name: prometheus-storage
@@ -7,7 +56,7 @@ metadata:
app.kubernetes.io/name: prometheus app.kubernetes.io/name: prometheus
spec: spec:
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
storageClassName: longhorn storageClassName: longhorn
resources: resources:
requests: requests:
@@ -26,75 +75,124 @@ data:
scrape_interval: 15s scrape_interval: 15s
scrape_timeout: 10s scrape_timeout: 10s
evaluation_interval: 15s evaluation_interval: 15s
scrape_configs: scrape_configs:
- job_name: 'prometheus' - job_name: prometheus
static_configs: static_configs:
- targets: ['localhost:9090'] - targets:
- localhost:9090
- job_name: 'vllm-qwen' - job_name: vllm-qwen
metrics_path: /metrics metrics_path: /metrics
static_configs: static_configs:
- targets: ['140.44.4.84:30800'] - targets:
labels: - 140.44.4.84:30800
model: 'Qwen2.5-7B-Instruct' labels:
node: 'snaisvr44hrvstr04' model: Qwen2.5-7B-Instruct
role: 'general-instruction' node: snaisvr44hrvstr04
role: general-instruction
- job_name: 'vllm-gemma4' - job_name: vllm-gemma4
metrics_path: /metrics metrics_path: /metrics
static_configs: static_configs:
- targets: ['140.44.4.82:30804'] - targets:
labels: - 140.44.4.82:30804
model: 'Gemma-4-12B-it' labels:
node: 'snaisvr44hrvstr02' model: Gemma-4-12B-it
role: 'coding-agent' node: snaisvr44hrvstr02
role: coding-agent
- job_name: 'vllm-deepseek-r1' - job_name: vllm-deepseek-r1
metrics_path: /metrics metrics_path: /metrics
static_configs: static_configs:
- targets: ['140.44.4.81:30805'] - targets:
labels: - 140.44.4.81:30805
model: 'DeepSeek-R1-Distill-Qwen-14B' labels:
node: 'snaisvr44hrvstr01' model: DeepSeek-R1-Distill-Qwen-14B
role: 'reasoning-planner' node: snaisvr44hrvstr01
role: reasoning-planner
- job_name: 'triton-inference-server' - job_name: triton-inference-server
metrics_path: /metrics metrics_path: /metrics
static_configs: static_configs:
- targets: ['140.44.4.83:30803'] - targets:
labels: - 140.44.4.83:30803
app: 'triton-inference-server' labels:
node: 'snaisvr44hrvstr03' app: triton-inference-server
model: 'decision_engine' node: snaisvr44hrvstr03
model: decision_engine
- job_name: 'qdrant' - job_name: qdrant
metrics_path: /metrics metrics_path: /metrics
static_configs: static_configs:
- targets: ['140.44.4.82:30633'] - targets:
labels: - 140.44.4.82:30633
app: 'qdrant' labels:
node: 'snaisvr44hrvstr02' app: qdrant
node: snaisvr44hrvstr02
- job_name: 'nvidia-dcgm' - job_name: nvidia-dcgm
metrics_path: /metrics metrics_path: /metrics
static_configs: static_configs:
- targets: ['140.44.4.81:9400'] - targets:
labels: - 140.44.4.81:9400
node: 'snaisvr44hrvstr01' labels:
gpu_model: 'Tesla T4' node: snaisvr44hrvstr01
- targets: ['140.44.4.82:9400'] gpu_model: Tesla T4
labels: - targets:
node: 'snaisvr44hrvstr02' - 140.44.4.82:9400
gpu_model: 'Tesla T4' labels:
- targets: ['140.44.4.83:9400'] node: snaisvr44hrvstr02
labels: gpu_model: Tesla T4
node: 'snaisvr44hrvstr03' - targets:
gpu_model: 'Tesla T4' - 140.44.4.83:9400
- targets: ['140.44.4.84:9400'] labels:
labels: node: snaisvr44hrvstr03
node: 'snaisvr44hrvstr04' gpu_model: Tesla T4
gpu_model: 'Tesla T4' - targets:
- 140.44.4.84:9400
labels:
node: snaisvr44hrvstr04
gpu_model: Tesla T4
- job_name: node-exporter
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.11:9100
- 140.44.4.12:9100
- 140.44.4.13:9100
- 140.44.4.21:9100
- 140.44.4.22:9100
- 140.44.4.23:9100
- 140.44.4.24:9100
- 140.44.4.25:9100
- 140.44.4.26:9100
- 140.44.4.27:9100
labels:
cluster: aerodrome
- targets:
- 140.44.4.41:9100
- 140.44.4.42:9100
- 140.44.4.43:9100
- 140.44.4.51:9100
- 140.44.4.52:9100
- 140.44.4.53:9100
- 140.44.4.54:9100
- 140.44.4.55:9100
- 140.44.4.56:9100
- 140.44.4.57:9100
labels:
cluster: spaceport
- job_name: kubernetes-cadvisor
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels:
- __meta_kubernetes_node_name
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
--- ---
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
@@ -120,50 +218,51 @@ spec:
runAsUser: 65534 runAsUser: 65534
runAsNonRoot: true runAsNonRoot: true
containers: containers:
- name: prometheus - name: prometheus
image: prom/prometheus:v2.54.1 image: prom/prometheus:v2.54.1
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
args: args:
- "--config.file=/etc/prometheus/prometheus.yml" - --config.file=/etc/prometheus/prometheus.yml
- "--storage.tsdb.path=/prometheus" - --storage.tsdb.path=/prometheus
- "--storage.tsdb.retention.time=90d" - --storage.tsdb.retention.time=90d
- "--web.enable-lifecycle" - --web.enable-lifecycle
- "--web.console.libraries=/usr/share/prometheus/console_libraries" - --web.console.libraries=/usr/share/prometheus/console_libraries
- "--web.console.templates=/usr/share/prometheus/consoles" - --web.console.templates=/usr/share/prometheus/consoles
ports: ports:
- name: web - name: web
containerPort: 9090 containerPort: 9090
resources: resources:
requests: requests:
cpu: "500m" cpu: 500m
memory: "1Gi" memory: 1Gi
limits: limits:
cpu: "2" cpu: '2'
memory: "4Gi" memory: 4Gi
readinessProbe: readinessProbe:
httpGet: httpGet:
path: /-/ready path: /-/ready
port: 9090 port: 9090
initialDelaySeconds: 10 initialDelaySeconds: 10
periodSeconds: 5 periodSeconds: 5
livenessProbe: livenessProbe:
httpGet: httpGet:
path: /-/healthy path: /-/healthy
port: 9090 port: 9090
initialDelaySeconds: 30 initialDelaySeconds: 30
periodSeconds: 15 periodSeconds: 15
volumeMounts: volumeMounts:
- name: config
mountPath: /etc/prometheus
- name: storage
mountPath: /prometheus
volumes:
- name: config - name: config
configMap: mountPath: /etc/prometheus
name: prometheus-config
- name: storage - name: storage
persistentVolumeClaim: mountPath: /prometheus
claimName: prometheus-storage volumes:
- name: config
configMap:
name: prometheus-config
- name: storage
persistentVolumeClaim:
claimName: prometheus-storage
serviceAccountName: prometheus
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
@@ -177,6 +276,6 @@ spec:
selector: selector:
app.kubernetes.io/name: prometheus app.kubernetes.io/name: prometheus
ports: ports:
- name: web - name: web
port: 9090 port: 9090
targetPort: 9090 targetPort: 9090