feat: add homelab deployment configs, documentation, and automation scripts

scripts/test_deploy_dcgm.py

@@ -0,0 +1,59 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
+    out = stdout.read().decode().strip()
+    err = stderr.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+manifest = """
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dcgm-exporter
+  namespace: ai-stack
+  labels:
+    app.kubernetes.io/name: dcgm-exporter
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: dcgm-exporter
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: dcgm-exporter
+    spec:
+      hostNetwork: true
+      containers:
+      - name: dcgm-exporter
+        image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
+        env:
+        - name: DCGM_EXPORTER_LISTEN
+          value: ":9400"
+        - name: DCGM_EXPORTER_KUBERNETES
+          value: "false"
+        securityContext:
+          privileged: true
+        ports:
+        - name: metrics
+          containerPort: 9400
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/dcgm.yaml")
+stdin.write(manifest)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying DCGM Exporter:")
+print(run("kubectl apply -f /tmp/dcgm.yaml"))
+import time
+time.sleep(5)
+print("Pods:")
+print(run("kubectl get pods -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o wide"))
+
+c.close()
+