Snarf/homelab-gitops
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Snarf:ed3884580297da3a56fb73d49cf9647e9b0711db
Branches Tags
Snarf:main
...
compare: Snarf:main
Branches Tags
Snarf:main

27 Commits
ed38845802 ... main

Author SHA1 Message Date
Snarf
5169a02440 docs(topology): regenerate high-res PNG diagram to reflect latest layout, hardware and storage metrics 2026-11-12 17:13:41 -05:00
Snarf
7381ddafd4 feat(homepage): update topology links to Authentik-protected endpoints on homepage.snarfnet.net 2026-11-12 17:08:50 -05:00
Snarf
c7eb47f6d8 docs(topology): include Drive count, Total Storage, and Harvester HCI storage in VLAN 40 2026-11-12 16:48:37 -05:00
Snarf
4ef2b40295 docs(topology): standardize Cluster A and B storage blocks to Storage Type and Storage Total 2026-11-12 16:22:40 -05:00
Snarf
63ab19172d docs(topology): standardize cluster headers with CPU Cores and Total RAM metrics 2026-11-12 16:15:11 -05:00
Snarf
a155a495fe docs(topology): update NAS hardware to Synology RS810+ and DS416play 2026-11-12 16:06:45 -05:00
Snarf
4297df615c revert(docs): restore previous homelab topology layout and architecture schematic 2026-11-12 15:58:17 -05:00
Snarf
d45c736dcd docs(topology): update homelab topology schematic with unified reference layout 2026-11-12 15:52:47 -05:00
Snarf
9cbc274033 fix(homepage): resolve broken logos for Argo CD and Tempest weather station 2026-11-12 15:39:42 -05:00
Snarf
620300f56f feat(homepage): add Homelab Topology service tile and bookmark links 2026-11-12 14:57:12 -05:00
Snarf
8b8fdeaf51 docs: add snarfnet homelab topology diagram html and static png 2026-11-12 14:55:23 -05:00
Snarf
911dfd2200 feat(homepage): add bookmarks for uptime kuma status pages (unified, core, gitops, edge) 2026-11-12 14:39:59 -05:00
Snarf
bf6aa90c63 fix(argocd): relocate raw dashboard json files to dashboards/ to resolve manifest generation error 2026-11-12 14:08:56 -05:00
Snarf
27d7ccdfdf feat(homepage): add direct links to grafana systems and container metrics dashboards 2026-11-12 13:14:23 -05:00
Snarf
efecce33db feat(grafana): provision systems performance and container metrics dashboards 2026-11-12 13:09:02 -05:00
Snarf
dfa2578723 feat(monitoring): enable node-exporter and kubernetes-cadvisor scraping in prometheus 2026-11-12 13:06:18 -05:00
Snarf
28f199f604 feat(homepage): style dashboard to match glass translucent theme and enable cluster & telemetry widgets 2026-11-12 11:59:19 -05:00
Snarf
bac981091c feat(homepage): add Rancher and Harvester cluster capacity widgets 2026-11-12 11:29:56 -05:00
Snarf
e65149381e feat(homepage): add cluster CPU/memory telemetry, Longhorn storage, and Portainer Swarm widgets 2026-11-12 11:06:21 -05:00
Snarf
3ddc7d01ab fix(homepage): format Tempest weather telemetry to tenths of a degree 2026-11-12 08:23:02 -05:00
Snarf
96e6b7a9b9 feat(ai-stack): add DCGM exporter, Prometheus scraper, and Grafana AI Stack intelligence dashboard 2026-11-11 13:51:19 -05:00
Snarf
9f0c3756ac fix(homepage): resolve widget errors, configure live Tempest HA entities, clean service cards 2026-11-11 08:30:48 -05:00
Snarf
7967eead8e feat(homepage): add dual longhorn, glances, rancher, portainer, and tempest weather widgets 2026-11-10 22:56:07 -05:00
Snarf
79e2032603 fix(uptime-kuma): route ingress directly to clusterip service http port and fix ai-stack node affinity 2026-11-10 13:45:36 -05:00
Snarf
7b2dd24526 feat: add ai-stack Kubernetes manifests for Qdrant, vLLM, and Triton 2026-11-08 15:03:24 -05:00
Snarf
be820c6b92 feat: add Kubernetes manifests for AI stack deployment including Qdrant, vLLM, and Triton 2026-11-08 14:56:54 -05:00
Snarf
a7c636cac0 feat: add Kubernetes manifests for AI stack deployment including Qdrant, vLLM, and Triton 2026-11-08 14:17:27 -05:00
13 changed files with 5212 additions and 37 deletions
Expand all files Collapse all files

28
argocd/aerodrome/prometheus-app.yaml Normal file
View File

@@ -0,0 +1,28 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: aerodrome-prometheus
namespace: infrastructure
labels:
app.kubernetes.io/name: aerodrome-prometheus
cluster: aerodrome
annotations:
notifications.argoproj.io/subscribe.on-sync-failed.gotify: ""
notifications.argoproj.io/subscribe.on-health-degraded.gotify: ""
spec:
project: homelab
source:
repoURL: "https://git.snarfnet.net/Snarf/homelab-gitops.git"
targetRevision: HEAD
path: manifests/aerodrome/prometheus
destination:
server: "https://140.44.4.11:6443"
namespace: default
syncPolicy:
automated:
prune: false
selfHeal: true
syncOptions:
- CreateNamespace=true
- ServerSideApply=true

523
dashboards/grafana/ai-stack-dashboard.json Normal file
View File

@@ -0,0 +1,523 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 100,
"title": "\u26a1 AI Stack Executive Overview & Cloud Cost Offsets",
"type": "row"
},
{
"id": 1,
"title": "Prompt Tokens Processed (In)",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum(vllm:prompt_tokens_total)",
"legendFormat": "Prompt Tokens",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
},
"unit": "locale"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 2,
"title": "Generation Tokens Synthesized (Out)",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum(vllm:generation_tokens_total)",
"legendFormat": "Gen Tokens",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "locale"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 3,
"title": "Homelab Cloud API Savings ($USD)",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)",
"legendFormat": "Net Savings",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "currencyUSD"
}
},
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
}
},
{
"id": 4,
"title": "Active GPU Cluster Health",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "count(DCGM_FI_DEV_GPU_TEMP)",
"legendFormat": "Tesla T4s Active",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
},
"unit": "short"
}
},
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center"
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 200,
"title": "\ud83c\udfae Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)",
"type": "row"
},
{
"id": 5,
"title": "GPU Core Utilization (%)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"datasource": "Prometheus",
"targets": [
{
"expr": "DCGM_FI_DEV_GPU_UTIL",
"legendFormat": "{{Hostname}} ({{modelName}})",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20
},
"unit": "percent",
"min": 0,
"max": 100
}
}
},
{
"id": 6,
"title": "VRAM Memory Allocated & Used",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"datasource": "Prometheus",
"targets": [
{
"expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024",
"legendFormat": "{{Hostname}} VRAM Used",
"refId": "A"
},
{
"expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024",
"legendFormat": "{{Hostname}} VRAM Total",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15
},
"unit": "bytes"
}
}
},
{
"id": 7,
"title": "GPU Thermal Profile (\u00b0C)",
"type": "timeseries",
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 14
},
"datasource": "Prometheus",
"targets": [
{
"expr": "DCGM_FI_DEV_GPU_TEMP",
"legendFormat": "{{Hostname}} (\u00b0C)",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10
},
"unit": "celsius",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 65
},
{
"color": "red",
"value": 80
}
]
}
}
}
},
{
"id": 8,
"title": "GPU Power Draw (Watts)",
"type": "timeseries",
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 14
},
"datasource": "Prometheus",
"targets": [
{
"expr": "DCGM_FI_DEV_POWER_USAGE",
"legendFormat": "{{Hostname}} (Watts)",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15
},
"unit": "watt"
}
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 21
},
"id": 300,
"title": "\ud83e\udde0 vLLM Real-Time Inference Dynamics & Token Velocity",
"type": "row"
},
{
"id": 9,
"title": "Token Generation Velocity (Tokens / Sec)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 22
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)",
"legendFormat": "{{model_name}} (Out)",
"refId": "A"
},
{
"expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)",
"legendFormat": "{{model_name}} (In)",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 25
},
"unit": "cps"
}
}
},
{
"id": 10,
"title": "Inference Concurrency & Request Backlog",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 22
},
"datasource": "Prometheus",
"targets": [
{
"expr": "vllm:num_requests_running",
"legendFormat": "{{model_name}} Running",
"refId": "A"
},
{
"expr": "vllm:num_requests_waiting",
"legendFormat": "{{model_name}} Queued",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "stepAfter",
"fillOpacity": 15
},
"unit": "short"
}
}
},
{
"id": 11,
"title": "GPU KV Cache Memory Saturation",
"type": "timeseries",
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 30
},
"datasource": "Prometheus",
"targets": [
{
"expr": "vllm:gpu_cache_usage_factor * 100",
"legendFormat": "{{model_name}} Cache Usage",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 30
},
"unit": "percent",
"min": 0,
"max": 100
}
}
},
{
"id": 12,
"title": "NVIDIA Triton & Qdrant Operations",
"type": "timeseries",
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 30
},
"datasource": "Prometheus",
"targets": [
{
"expr": "rate(nv_inference_request_success[2m])",
"legendFormat": "Triton Success/s",
"refId": "A"
},
{
"expr": "cpu_cores_used",
"legendFormat": "Qdrant CPU Cores",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth"
},
"unit": "short"
}
}
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": [
"ai",
"vllm",
"dcgm",
"gpu",
"snarfnet"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m"
]
},
"timezone": "browser",
"title": "SnarfNet AI Stack & Inference Intelligence",
"uid": "snarfnet-ai-stack",
"version": 1
}

476
dashboards/grafana/container-metrics-dashboard.json Normal file
View File

@@ -0,0 +1,476 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 100,
"title": "\ud83d\udce6 Container Fleet & Pod Metrics",
"type": "row"
},
{
"id": 1,
"title": "Monitored Containers",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "count(container_last_seen{container!=''})",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 2,
"title": "Active Kubernetes Pods",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "count(count by (pod, namespace) (container_last_seen{container!=''}))",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
},
"unit": "none"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 3,
"title": "Total Container CPU Load",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
},
"unit": "none"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 4,
"title": "Total Container RAM",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum(container_memory_working_set_bytes{container!=''})",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "orange",
"value": null
}
]
},
"unit": "bytes"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 200,
"title": "\u26a1 Top Container CPU Utilization",
"type": "row"
},
{
"id": 5,
"title": "Top 10 CPU Consuming Pods (Cores)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"datasource": "Prometheus",
"targets": [
{
"expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))",
"legendFormat": "{{namespace}} / {{pod}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "none"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 6,
"title": "CPU Throttling by Namespace",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))",
"legendFormat": "{{namespace}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "none"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 300,
"title": "\ud83e\udde0 Container Memory Working Set",
"type": "row"
},
{
"id": 7,
"title": "Top 10 Memory Consuming Pods",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 14,
"x": 0,
"y": 15
},
"datasource": "Prometheus",
"targets": [
{
"expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))",
"legendFormat": "{{namespace}} / {{pod}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "bytes"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 8,
"title": "Memory by Namespace",
"type": "bargauge",
"gridPos": {
"h": 8,
"w": 10,
"x": 14,
"y": 15
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum by (namespace) (container_memory_working_set_bytes{container!=''})",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
},
"unit": "bytes",
"min": 0,
"max": 100
}
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"showUnfilled": true
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 400,
"title": "\ud83c\udf10 Container Network Traffic",
"type": "row"
},
{
"id": 9,
"title": "Top Pod Network RX (Receive)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"datasource": "Prometheus",
"targets": [
{
"expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))",
"legendFormat": "RX {{namespace}}/{{pod}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "Bps"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 10,
"title": "Top Pod Network TX (Transmit)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"datasource": "Prometheus",
"targets": [
{
"expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))",
"legendFormat": "TX {{namespace}}/{{pod}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "Bps"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": [
"kubernetes",
"containers",
"cadvisor",
"snarfnet",
"workloads"
],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "\ud83d\udce6 Kubernetes Containers & Workloads",
"uid": "snarfnet-containers"
}

595
dashboards/grafana/systems-dashboard.json Normal file
View File

@@ -0,0 +1,595 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 100,
"title": "\ud83d\udda5\ufe0f Systems Fleet Overview & Capacity",
"type": "row"
},
{
"id": 1,
"title": "Online Node Exporters",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "count(node_load1)",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 2,
"title": "Avg Cluster CPU Load (1m)",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "avg(node_load1)",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
},
"unit": "none"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 3,
"title": "Total Monitored RAM",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum(node_memory_MemTotal_bytes)",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
},
"unit": "bytes"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"id": 4,
"title": "Fleet RAM In-Use %",
"type": "stat",
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 1
},
"datasource": "Prometheus",
"targets": [
{
"expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "orange",
"value": null
}
]
},
"unit": "percent"
}
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center"
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 200,
"title": "\u26a1 CPU Load & Core Utilization",
"type": "row"
},
{
"id": 5,
"title": "CPU Utilization by Host (%)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"datasource": "Prometheus",
"targets": [
{
"expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)",
"legendFormat": "{{cluster}} - {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "percent"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 6,
"title": "1-Minute Load Average by Host",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"datasource": "Prometheus",
"targets": [
{
"expr": "node_load1",
"legendFormat": "{{cluster}} - {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "none"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 300,
"title": "\ud83e\udde0 Memory Consumption & Free Capacity",
"type": "row"
},
{
"id": 7,
"title": "Memory Used % by Host",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"datasource": "Prometheus",
"targets": [
{
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
"legendFormat": "{{cluster}} - {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "percent"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 8,
"title": "Available RAM by Host (GiB)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"datasource": "Prometheus",
"targets": [
{
"expr": "node_memory_MemAvailable_bytes",
"legendFormat": "{{cluster}} - {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "bytes"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 400,
"title": "\ud83d\udcbe Filesystem & Disk I/O Performance",
"type": "row"
},
{
"id": 9,
"title": "Root (/) Filesystem Usage %",
"type": "bargauge",
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 24
},
"datasource": "Prometheus",
"targets": [
{
"expr": "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
},
"unit": "percent",
"min": 0,
"max": 100
}
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"showUnfilled": true
}
},
{
"id": 10,
"title": "Disk Read Throughput (Bytes/sec)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 24
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))",
"legendFormat": "Read {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "Bps"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 11,
"title": "Disk Write Throughput (Bytes/sec)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 24
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))",
"legendFormat": "Write {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "Bps"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 32
},
"id": 500,
"title": "\ud83c\udf10 Network Throughput (RX / TX)",
"type": "row"
},
{
"id": 12,
"title": "Host Network Receive (RX)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 33
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))",
"legendFormat": "RX {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "Bps"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 13,
"title": "Host Network Transmit (TX)",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 33
},
"datasource": "Prometheus",
"targets": [
{
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))",
"legendFormat": "TX {{instance}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2,
"fillOpacity": 15
},
"unit": "Bps"
}
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": [
"infrastructure",
"systems",
"snarfnet",
"node-exporter"
],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "\ud83d\udda5\ufe0f Systems & Cluster Host Performance",
"uid": "snarfnet-systems"
}

1029
docs/snarfnet_homelab_topology.html Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
docs/snarfnet_homelab_topology.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 830 KiB

3
manifests/aerodrome/grafana/.argocdignore Normal file
View File

@@ -0,0 +1,3 @@
*.json
*.md

1698
manifests/aerodrome/grafana/grafana.yaml
View File

File diff suppressed because it is too large Load Diff

281
manifests/aerodrome/prometheus/prometheus.yaml Normal file
View File

@@ -0,0 +1,281 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: default
labels:
app.kubernetes.io/name: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
rules:
- apiGroups:
- ''
resources:
- nodes
- nodes/proxy
- nodes/metrics
- services
- endpoints
- pods
verbs:
- get
- list
- watch
- nonResourceURLs:
- /metrics
- /metrics/cadvisor
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: default
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-storage
namespace: default
labels:
app.kubernetes.io/name: prometheus
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 50Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: default
labels:
app.kubernetes.io/name: prometheus
data:
prometheus.yml: |
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- job_name: vllm-qwen
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.84:30800
labels:
model: Qwen2.5-7B-Instruct
node: snaisvr44hrvstr04
role: general-instruction
- job_name: vllm-gemma4
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.82:30804
labels:
model: Gemma-4-12B-it
node: snaisvr44hrvstr02
role: coding-agent
- job_name: vllm-deepseek-r1
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.81:30805
labels:
model: DeepSeek-R1-Distill-Qwen-14B
node: snaisvr44hrvstr01
role: reasoning-planner
- job_name: triton-inference-server
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.83:30803
labels:
app: triton-inference-server
node: snaisvr44hrvstr03
model: decision_engine
- job_name: qdrant
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.82:30633
labels:
app: qdrant
node: snaisvr44hrvstr02
- job_name: nvidia-dcgm
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.81:9400
labels:
node: snaisvr44hrvstr01
gpu_model: Tesla T4
- targets:
- 140.44.4.82:9400
labels:
node: snaisvr44hrvstr02
gpu_model: Tesla T4
- targets:
- 140.44.4.83:9400
labels:
node: snaisvr44hrvstr03
gpu_model: Tesla T4
- targets:
- 140.44.4.84:9400
labels:
node: snaisvr44hrvstr04
gpu_model: Tesla T4
- job_name: node-exporter
metrics_path: /metrics
static_configs:
- targets:
- 140.44.4.11:9100
- 140.44.4.12:9100
- 140.44.4.13:9100
- 140.44.4.21:9100
- 140.44.4.22:9100
- 140.44.4.23:9100
- 140.44.4.24:9100
- 140.44.4.25:9100
- 140.44.4.26:9100
- 140.44.4.27:9100
labels:
cluster: aerodrome
- targets:
- 140.44.4.41:9100
- 140.44.4.42:9100
- 140.44.4.43:9100
- 140.44.4.51:9100
- 140.44.4.52:9100
- 140.44.4.53:9100
- 140.44.4.54:9100
- 140.44.4.55:9100
- 140.44.4.56:9100
- 140.44.4.57:9100
labels:
cluster: spaceport
- job_name: kubernetes-cadvisor
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels:
- __meta_kubernetes_node_name
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
namespace: default
labels:
app.kubernetes.io/name: prometheus
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: prometheus
template:
metadata:
labels:
app.kubernetes.io/name: prometheus
spec:
securityContext:
fsGroup: 65534
runAsUser: 65534
runAsNonRoot: true
containers:
- name: prometheus
image: prom/prometheus:v2.54.1
imagePullPolicy: IfNotPresent
args:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=90d
- --web.enable-lifecycle
- --web.console.libraries=/usr/share/prometheus/console_libraries
- --web.console.templates=/usr/share/prometheus/consoles
ports:
- name: web
containerPort: 9090
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: '2'
memory: 4Gi
readinessProbe:
httpGet:
path: /-/ready
port: 9090
initialDelaySeconds: 10
periodSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: 9090
initialDelaySeconds: 30
periodSeconds: 15
volumeMounts:
- name: config
mountPath: /etc/prometheus
- name: storage
mountPath: /prometheus
volumes:
- name: config
configMap:
name: prometheus-config
- name: storage
persistentVolumeClaim:
claimName: prometheus-storage
serviceAccountName: prometheus
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: default
labels:
app.kubernetes.io/name: prometheus
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: prometheus
ports:
- name: web
port: 9090
targetPort: 9090

5
manifests/aerodrome/uptime-kuma/uptime-kuma.yaml
View File

@@ -115,6 +115,7 @@ kind: Ingress
metadata: metadata:
annotations: annotations:
traefik.ingress.kubernetes.io/router.middlewares: monitoring-authentik-forwardauth@kubernetescrd traefik.ingress.kubernetes.io/router.middlewares: monitoring-authentik-forwardauth@kubernetescrd
traefik.ingress.kubernetes.io/service.serversscheme: http
labels: labels:
app.kubernetes.io/name: uptime-kuma app.kubernetes.io/name: uptime-kuma
name: uptime-ing name: uptime-ing
@@ -127,9 +128,9 @@ spec:
paths: paths:
- backend: - backend:
service: service:
name: uptime-lb name: uptime-kuma-2-1756668961
port: port:
number: 448 number: 3001
path: / path: /
pathType: ImplementationSpecific pathType: ImplementationSpecific
tls: tls:

286
manifests/spacedock/ai-stack/ai-stack.yaml
View File

@@ -126,6 +126,8 @@ spec:
labels: labels:
app: vllm-openai app: vllm-openai
spec: spec:
nodeSelector:
kubernetes.io/hostname: snaisvr44hrvstr04
containers: containers:
- name: vllm - name: vllm
image: vllm/vllm-openai:v0.7.3 image: vllm/vllm-openai:v0.7.3
@@ -136,11 +138,11 @@ spec:
- "--served-model-name" - "--served-model-name"
- "Qwen2.5-7B-Instruct" - "Qwen2.5-7B-Instruct"
- "--quantization" - "--quantization"
- "awq_marlin" - "awq"
- "--dtype" - "--dtype"
- "half" - "half"
- "--max-model-len" - "--max-model-len"
- "16384" - "65536"
- "--gpu-memory-utilization" - "--gpu-memory-utilization"
- "0.90" - "0.90"
- "--enable-auto-tool-choice" - "--enable-auto-tool-choice"
@@ -155,6 +157,8 @@ spec:
value: "/root/.cache/huggingface" value: "/root/.cache/huggingface"
- name: VLLM_WORKER_MULTIPROC_METHOD - name: VLLM_WORKER_MULTIPROC_METHOD
value: "spawn" value: "spawn"
- name: VLLM_ALLOW_LONG_MAX_MODEL_LEN
value: "1"
ports: ports:
- name: http-openai - name: http-openai
containerPort: 8000 containerPort: 8000
@@ -205,6 +209,234 @@ spec:
nodePort: 30800 nodePort: 30800
--- ---
# ============================================================================== # ==============================================================================
# 2B. vLLM GEMMA 4 12B INFERENCE ENGINE (snaisvr44hrvstr02 Tesla T4 GPU)
# ==============================================================================
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: vllm-gemma4-models-pvc
namespace: ai-stack
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 200Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: vllm-gemma4
namespace: ai-stack
labels:
app: vllm-gemma4
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: vllm-gemma4
template:
metadata:
labels:
app: vllm-gemma4
spec:
nodeSelector:
kubernetes.io/hostname: snaisvr44hrvstr02
containers:
- name: vllm
image: vllm/vllm-openai:v0.32.0-cu129
imagePullPolicy: IfNotPresent
command: ["/bin/bash", "-c"]
args:
- |
rm -rf /usr/local/lib/python3.12/dist-packages/torchcodec*
python3 -c '
p = "/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/triton_unified_attention.py"
s = open(p).read()
s = s.replace("return 32", "return 16")
s = s.replace("launch_num_stages: int | None = None", "launch_num_stages: int | None = 1")
open(p, "w").write(s)
'
exec vllm serve google/gemma-4-12B-it-qat-w4a16-ct \
--served-model-name Gemma-4-12B-it \
--dtype half \
--max-model-len 8192 \
--gpu-memory-utilization 0.92 \
--enforce-eager \
--enable-auto-tool-choice \
--tool-call-parser gemma4 \
--reasoning-parser gemma4 \
--host 0.0.0.0 \
--port 8000
env:
- name: HF_HOME
value: "/root/.cache/huggingface"
- name: VLLM_WORKER_MULTIPROC_METHOD
value: "spawn"
- name: VLLM_ALLOW_LONG_MAX_MODEL_LEN
value: "1"
ports:
- name: http-openai
containerPort: 8000
resources:
requests:
cpu: "8"
memory: "24Gi"
nvidia.com/gpu: "1"
limits:
cpu: "24"
memory: "48Gi"
nvidia.com/gpu: "1"
readinessProbe:
httpGet:
path: /health
port: 8000
initialDelaySeconds: 30
periodSeconds: 10
volumeMounts:
- name: vllm-gemma4-models
mountPath: /root/.cache/huggingface
- name: dshm
mountPath: /dev/shm
volumes:
- name: vllm-gemma4-models
persistentVolumeClaim:
claimName: vllm-gemma4-models-pvc
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 16Gi
---
apiVersion: v1
kind: Service
metadata:
name: vllm-gemma4
namespace: ai-stack
labels:
app: vllm-gemma4
spec:
type: NodePort
selector:
app: vllm-gemma4
ports:
- name: http-openai
port: 8000
targetPort: 8000
nodePort: 30804
---
# ==============================================================================
# 2C. DEEPSEEK-R1-DISTILL-QWEN-14B REASONING ENGINE (snaisvr44hrvstr01 Tesla T4 GPU)
# ==============================================================================
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: glm-superexpert-models-pvc
namespace: ai-stack
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 200Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: glm-superexpert
namespace: ai-stack
labels:
app: glm-superexpert
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: glm-superexpert
template:
metadata:
labels:
app: glm-superexpert
spec:
nodeSelector:
kubernetes.io/hostname: snaisvr44hrvstr01
containers:
- name: vllm
image: vllm/vllm-openai:v0.32.0-cu129
imagePullPolicy: IfNotPresent
command: ["/bin/bash", "-c"]
args:
- |
rm -rf /usr/local/lib/python3.12/dist-packages/torchcodec*
exec vllm serve casperhansen/deepseek-r1-distill-qwen-14b-awq \
--served-model-name DeepSeek-R1-Distill-Qwen-14B \
--quantization awq \
--dtype half \
--max-model-len 16384 \
--gpu-memory-utilization 0.92 \
--enforce-eager \
--enable-auto-tool-choice \
--tool-call-parser hermes \
--reasoning-parser deepseek_r1 \
--host 0.0.0.0 \
--port 8000
env:
- name: HF_HOME
value: "/root/.cache/huggingface"
ports:
- name: http-openai
containerPort: 8000
resources:
requests:
cpu: "8"
memory: "20Gi"
nvidia.com/gpu: "1"
limits:
cpu: "24"
memory: "40Gi"
nvidia.com/gpu: "1"
readinessProbe:
httpGet:
path: /health
port: 8000
initialDelaySeconds: 30
periodSeconds: 10
volumeMounts:
- name: glm-models
mountPath: /root/.cache/huggingface
- name: dshm
mountPath: /dev/shm
volumes:
- name: glm-models
persistentVolumeClaim:
claimName: glm-superexpert-models-pvc
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 16Gi
---
apiVersion: v1
kind: Service
metadata:
name: glm-superexpert
namespace: ai-stack
labels:
app: glm-superexpert
spec:
type: NodePort
selector:
app: glm-superexpert
ports:
- name: http-openai
port: 8000
targetPort: 8000
nodePort: 30805
---
# ==============================================================================
# 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving) # 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving)
# ============================================================================== # ==============================================================================
apiVersion: v1 apiVersion: v1
@@ -377,3 +609,53 @@ spec:
port: 8002 port: 8002
targetPort: 8002 targetPort: 8002
nodePort: 30803 nodePort: 30803
---
# ==============================================================================
# 4. NVIDIA DCGM EXPORTER (Tesla T4 Telemetry: Utilization, VRAM, Power, Temp)
# ==============================================================================
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: dcgm-exporter
namespace: ai-stack
labels:
app.kubernetes.io/name: dcgm-exporter
spec:
selector:
matchLabels:
app.kubernetes.io/name: dcgm-exporter
template:
metadata:
labels:
app.kubernetes.io/name: dcgm-exporter
spec:
hostNetwork: true
containers:
- name: dcgm-exporter
image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
env:
- name: DCGM_EXPORTER_LISTEN
value: ":9400"
- name: DCGM_EXPORTER_KUBERNETES
value: "false"
- name: LD_LIBRARY_PATH
value: "/usr/local/nvidia/lib64"
securityContext:
privileged: true
ports:
- name: metrics
containerPort: 9400
volumeMounts:
- name: dev
mountPath: /dev
- name: nvidia-lib
mountPath: /usr/local/nvidia/lib64
readOnly: true
volumes:
- name: dev
hostPath:
path: /dev
- name: nvidia-lib
hostPath:
path: /usr/local/nvidia/lib64

291
manifests/spaceport/homepage/homepage.yaml
View File

@@ -147,39 +147,6 @@ spec:
type: ClusterIP type: ClusterIP
--- ---
apiVersion: v1 apiVersion: v1
data:
bookmarks.yaml: "- Developer:\n - Github:\n - abbr: GH\n href: https://github.com/\n"
docker.yaml: ''
kubernetes.yaml: 'mode: disable
'
services.yaml: "- Critical Services:\n - Authentik:\n icon: authentik.png\n\
\ href: https://authentik.snarfnet.net\n widget:\n type: authentik\n\
\ url: https://authentik.snarfnet.net\n fields: [\"users\", \"loginsLast24H\"\
, \"failedLoginsLast24H\"]\n key: 7HamapUhYizcNudHDIT5IFy3XEpRgeKgqr1Pw4ub9eTpAjMAzB8Aur1Z6dUU\n\
\ - PeaNUT UPS:\n icon: peanut.png\n href: https://portal.snarfnet.net\n\
\ widget:\n type: peanut\n url: https://peanut.snarfnet.net\n\
\ fields: [\"battery_charge\", \"ups_load\", \"ups_status\"]\n key:\
\ server-closet\n - Unifi Controller:\n icon: unifi.png\n href: https://unifi.ui.com\n\
\ widget:\n type: unifi\n url: https://140.44.44.1\n username:\
\ Snarf\n password: Katzir476!!!\n- Git Ops:\n - Gitea:\n icon: gitea.png\n\
\ href: https://git.snarfnet.net\n widget:\n type: gitea\n \
\ url: https://git.snarfnet.net\n fields: [\"repositories\", \"notifications\"\
, \"issues\", \"pulls\"]\n key: 47c10c54574131c007302b246352f653facb5062\n\
- Infrastructure:\n - NGINX proxy manager:\n icon: nginx-proxy-manager\n\
\ href: https://nginx.snarfnet.net\n widget:\n type: npm\n \
\ url: https://nginx.snarfnet.net\n fields: [\"enabled\", \"disabled\"\
, \"total\"]\n username: snarf@snarfnet.net\n password: Katzir476!\n\
\ - Uptime Kuma:\n icon: uptime-kuma\n href: https://uptime.snarfnet.net\n\
\ widget:\n type: uptimekuma\n url: https://uptime.snarfnet.net\n\
\ fields: [\"up\", \"down\", \"uptime\", \"incident\"]\n slug: public\n"
settings.yaml: "title: SnarfNet Landing Zone\nheaderStyle: boxedWidgets\ncolor:\
\ green\ntheme: dark\nlayout:\n Critical Services:\n Git Ops:\n Infrastructure:\n\
showStats: true\nproviders:\n longhorn:\n url: https://longhorn.aerodrome.snarfnet.net\n"
widgets.yaml: "- resources:\n backend: resources\n cpu: true\n expanded:\
\ true\n memory: true\n- search:\n provider: duckduckgo\n target: _blank\n\
- longhorn:\n expanded: true\n total: true\n labels: true\n nodes:\
\ true\n"
kind: ConfigMap kind: ConfigMap
metadata: metadata:
annotations: annotations:
@@ -193,3 +160,261 @@ metadata:
helm.sh/chart: homepage-2.1.0 helm.sh/chart: homepage-2.1.0
name: homepage-2-1744068158 name: homepage-2-1744068158
namespace: infrastructure namespace: infrastructure
data:
bookmarks.yaml: |
- Quick Links:
- GitHub:
- abbr: GH
href: https://github.com/
- UniFi Site Manager:
- abbr: UI
href: https://unifi.ui.com/
- Cloudflare:
- abbr: CF
href: https://dash.cloudflare.com/
- Grafana Telemetry:
- Systems Performance:
- abbr: SYS
href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861
- Container Workloads:
- abbr: K8S
href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads
- AI Stack Intelligence:
- abbr: AI
href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence
- Uptime Status Pages:
- Unified Overview:
- abbr: ALL
href: https://uptime.snarfnet.net/status/default
- Core & VIP Fabric:
- abbr: VIP
href: https://uptime.snarfnet.net/status/core
- GitOps & CI/CD:
- abbr: OPS
href: https://uptime.snarfnet.net/status/gitops
- Edge & Ingress:
- abbr: EDG
href: https://uptime.snarfnet.net/status/edge
- Homelab Architecture:
- Topology Diagram (Image):
- abbr: IMG
href: https://homepage.snarfnet.net/topology.png
- Interactive Topology (HTML):
- abbr: MAP
href: https://homepage.snarfnet.net/topology.html
docker.yaml: ''
kubernetes.yaml: |
mode: cluster
services.yaml: |
- Critical Services:
- Authentik:
icon: authentik.png
href: https://authentik.snarfnet.net
description: SSO & Identity Management
- PeaNUT UPS:
icon: peanut.png
href: https://portal.snarfnet.net
description: UPS Power & Battery Monitor
- Unifi Controller:
icon: unifi.png
href: https://unifi.ui.com
widget:
type: unifi
url: https://140.44.44.1
username: Snarf
password: Katzir476!!!
- GitOps & Automation:
- Gitea:
icon: gitea.png
href: https://git.snarfnet.net
description: Git Repositories
widget:
type: gitea
url: https://git.snarfnet.net
key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71
- ArgoCD:
icon: argo-cd.png
href: https://argo.snarfnet.net
description: Declarative GitOps Reconciler
widget:
type: argocd
url: https://argo.snarfnet.net
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0
- AWX Automation:
icon: awx.png
href: https://awx.snarfnet.net
description: Ansible Automation Controller
- Semaphore:
icon: semaphore.png
href: https://semaphore.snarfnet.net
description: Playbook Execution Engine
- Storage & Clusters:
- Longhorn Aerodrome:
icon: longhorn.png
href: https://longhorn.aerodrome.snarfnet.net
description: Aerodrome Distributed Flash Storage
- Longhorn Spaceport:
icon: longhorn.png
href: https://longhorn.spaceport.snarfnet.net
description: Spaceport Distributed Block Storage
- Rancher (Aerodrome):
icon: rancher.png
href: https://aerodrome.snarfnet.net
description: Kubernetes Management Console
widget:
type: customapi
url: https://aerodrome.snarfnet.net/v3/clusters/local
headers:
Authorization: Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7
mappings:
- field: capacity.cpu
label: CPU Cores
- field: capacity.pods
label: Max Pods
- field: state
label: Cluster State
- Harvester HCI (Spacedock):
icon: harvester.png
href: https://harvester.spacedock.snarfnet.net
description: HCI Virtualization Hypervisors
widget:
type: customapi
url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt
headers:
Authorization: Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7
mappings:
- field: capacity.cpu
label: CPU Cores
- field: capacity.pods
label: Max Pods
- field: state
label: HCI State
- Portainer Swarm:
icon: portainer.png
href: https://shipyard.snarfnet.net
description: Docker Swarm Container Fleet
widget:
type: portainer
url: https://140.44.4.71:9443
env: 3
key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8=
- Infrastructure & Observability:
- NGINX Proxy Manager:
icon: nginx-proxy-manager
href: https://nginx.snarfnet.net
description: Reverse Proxy & SSL Certificates
- Uptime Kuma:
icon: uptime-kuma
href: https://uptime.snarfnet.net
description: Multi-Endpoint Availability Probes
widget:
type: uptimekuma
url: http://140.44.4.28:448
slug: default
- Zabbix:
icon: zabbix.png
href: https://zabbix.snarfnet.net
description: Infrastructure Metrics & Alarms
- Grafana:
icon: grafana.png
href: https://grafana.snarfnet.net
description: Real-Time Metrics & AI Dashboards
widget:
type: grafana
url: https://grafana.snarfnet.net
username: admin
password: Katzir476!!!
- Core Node Glances:
icon: glances.png
href: http://140.44.4.11:61208
description: Host Resource Monitor
- Homelab Topology:
icon: mdi-sitemap
href: https://homepage.snarfnet.net/topology.html
description: Architecture & Network Schematic
- Environment & Weather:
- Tempest Weather Station:
icon: mdi-weather-partly-cloudy
href: https://snarfhome.snarfnet.net
description: Backyard Weather Telemetry
widget:
type: homeassistant
url: http://14.4.4.20:8123
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4
custom:
- template: '{{ states(''sensor.st_00158555_temperature'') | float(0) | round(1)
}} °F'
label: Outdoor Temp
- template: '{{ states(''sensor.st_00158555_feels_like'') | float(0) | round(1)
}} °F'
label: Feels Like
- template: '{{ states(''sensor.st_00158555_humidity'') | float(0) | round(1)
}} %'
label: Humidity
- template: '{{ states(''sensor.st_00158555_wind_speed'') | float(0) | round(1)
}} mph'
label: Wind Speed
settings.yaml: |
title: SnarfNet Operations & Telemetry
headerStyle: boxed
color: slate
theme: dark
statusStyle: badge
background:
image: https://i.imgur.com/hOzzkCY.jpeg
blur: sm
opacity: 50
brightness: 50
cardBlur: sm
quicklaunch:
searchDescriptions: true
hideInternetSearch: true
hideVisitURL: true
hideVersion: true
layout:
Critical Services:
icon: mdi-shield-check
style: row
columns: 3
GitOps & Automation:
icon: mdi-git
style: row
columns: 4
Storage & Clusters:
icon: mdi-server-network
style: row
columns: 3
Infrastructure & Observability:
icon: mdi-chart-timeline-variant
style: row
columns: 3
Environment & Weather:
icon: mdi-weather-partly-cloudy
style: row
columns: 1
showStats: true
providers:
longhorn:
url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500
widgets.yaml: |
- kubernetes:
cluster:
show: true
cpu: true
memory: true
show_names: true
nodes:
show: false
- longhorn:
total: true
nodes: false
- resources:
label: System
backend: resources
cpu: true
expanded: true
memory: true
disk: /
- search:
provider: duckduckgo
target: _blank

34
manifests/spaceport/homepage/rbac.yaml Normal file
View File

@@ -0,0 +1,34 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: homepage-reader
rules:
- apiGroups: [""]
resources: ["nodes", "namespaces", "pods", "services", "endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingresses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["metrics.k8s.io"]
resources: ["nodes", "pods"]
verbs: ["get", "list"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "list"]
- apiGroups: ["traefik.containo.us", "traefik.io"]
resources: ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "traefikservices"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: homepage-reader-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: homepage-reader
subjects:
- kind: ServiceAccount
name: default
namespace: infrastructure