282 lines
6.5 KiB
YAML
282 lines
6.5 KiB
YAML
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: prometheus
|
|
namespace: default
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: prometheus
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
rules:
|
|
- apiGroups:
|
|
- ''
|
|
resources:
|
|
- nodes
|
|
- nodes/proxy
|
|
- nodes/metrics
|
|
- services
|
|
- endpoints
|
|
- pods
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- nonResourceURLs:
|
|
- /metrics
|
|
- /metrics/cadvisor
|
|
verbs:
|
|
- get
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: prometheus
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: prometheus
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: prometheus
|
|
namespace: default
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: prometheus-storage
|
|
namespace: default
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 50Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
namespace: default
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 15s
|
|
scrape_timeout: 10s
|
|
evaluation_interval: 15s
|
|
scrape_configs:
|
|
- job_name: prometheus
|
|
static_configs:
|
|
- targets:
|
|
- localhost:9090
|
|
- job_name: vllm-qwen
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.84:30800
|
|
labels:
|
|
model: Qwen2.5-7B-Instruct
|
|
node: snaisvr44hrvstr04
|
|
role: general-instruction
|
|
- job_name: vllm-gemma4
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.82:30804
|
|
labels:
|
|
model: Gemma-4-12B-it
|
|
node: snaisvr44hrvstr02
|
|
role: coding-agent
|
|
- job_name: vllm-deepseek-r1
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.81:30805
|
|
labels:
|
|
model: DeepSeek-R1-Distill-Qwen-14B
|
|
node: snaisvr44hrvstr01
|
|
role: reasoning-planner
|
|
- job_name: triton-inference-server
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.83:30803
|
|
labels:
|
|
app: triton-inference-server
|
|
node: snaisvr44hrvstr03
|
|
model: decision_engine
|
|
- job_name: qdrant
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.82:30633
|
|
labels:
|
|
app: qdrant
|
|
node: snaisvr44hrvstr02
|
|
- job_name: nvidia-dcgm
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.81:9400
|
|
labels:
|
|
node: snaisvr44hrvstr01
|
|
gpu_model: Tesla T4
|
|
- targets:
|
|
- 140.44.4.82:9400
|
|
labels:
|
|
node: snaisvr44hrvstr02
|
|
gpu_model: Tesla T4
|
|
- targets:
|
|
- 140.44.4.83:9400
|
|
labels:
|
|
node: snaisvr44hrvstr03
|
|
gpu_model: Tesla T4
|
|
- targets:
|
|
- 140.44.4.84:9400
|
|
labels:
|
|
node: snaisvr44hrvstr04
|
|
gpu_model: Tesla T4
|
|
- job_name: node-exporter
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- 140.44.4.11:9100
|
|
- 140.44.4.12:9100
|
|
- 140.44.4.13:9100
|
|
- 140.44.4.21:9100
|
|
- 140.44.4.22:9100
|
|
- 140.44.4.23:9100
|
|
- 140.44.4.24:9100
|
|
- 140.44.4.25:9100
|
|
- 140.44.4.26:9100
|
|
- 140.44.4.27:9100
|
|
labels:
|
|
cluster: aerodrome
|
|
- targets:
|
|
- 140.44.4.41:9100
|
|
- 140.44.4.42:9100
|
|
- 140.44.4.43:9100
|
|
- 140.44.4.51:9100
|
|
- 140.44.4.52:9100
|
|
- 140.44.4.53:9100
|
|
- 140.44.4.54:9100
|
|
- 140.44.4.55:9100
|
|
- 140.44.4.56:9100
|
|
- 140.44.4.57:9100
|
|
labels:
|
|
cluster: spaceport
|
|
- job_name: kubernetes-cadvisor
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: kubernetes.default.svc:443
|
|
- source_labels:
|
|
- __meta_kubernetes_node_name
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: prometheus
|
|
namespace: default
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: prometheus
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 65534
|
|
runAsUser: 65534
|
|
runAsNonRoot: true
|
|
containers:
|
|
- name: prometheus
|
|
image: prom/prometheus:v2.54.1
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --config.file=/etc/prometheus/prometheus.yml
|
|
- --storage.tsdb.path=/prometheus
|
|
- --storage.tsdb.retention.time=90d
|
|
- --web.enable-lifecycle
|
|
- --web.console.libraries=/usr/share/prometheus/console_libraries
|
|
- --web.console.templates=/usr/share/prometheus/consoles
|
|
ports:
|
|
- name: web
|
|
containerPort: 9090
|
|
resources:
|
|
requests:
|
|
cpu: 500m
|
|
memory: 1Gi
|
|
limits:
|
|
cpu: '2'
|
|
memory: 4Gi
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /-/ready
|
|
port: 9090
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 5
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /-/healthy
|
|
port: 9090
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 15
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/prometheus
|
|
- name: storage
|
|
mountPath: /prometheus
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: prometheus-config
|
|
- name: storage
|
|
persistentVolumeClaim:
|
|
claimName: prometheus-storage
|
|
serviceAccountName: prometheus
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: prometheus
|
|
namespace: default
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app.kubernetes.io/name: prometheus
|
|
ports:
|
|
- name: web
|
|
port: 9090
|
|
targetPort: 9090
|