75 lines
2.0 KiB
Python
75 lines
2.0 KiB
Python
import paramiko
|
|
import time
|
|
|
|
c = paramiko.SSHClient()
|
|
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
|
|
|
def run(cmd):
|
|
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
|
|
out = stdout.read().decode().strip()
|
|
return out
|
|
|
|
manifest = """
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: dcgm-exporter
|
|
namespace: ai-stack
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
spec:
|
|
hostNetwork: true
|
|
containers:
|
|
- name: dcgm-exporter
|
|
image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
|
|
env:
|
|
- name: DCGM_EXPORTER_LISTEN
|
|
value: ":9400"
|
|
- name: DCGM_EXPORTER_KUBERNETES
|
|
value: "false"
|
|
- name: LD_LIBRARY_PATH
|
|
value: "/usr/local/nvidia/lib64"
|
|
securityContext:
|
|
privileged: true
|
|
ports:
|
|
- name: metrics
|
|
containerPort: 9400
|
|
volumeMounts:
|
|
- name: dev
|
|
mountPath: /dev
|
|
- name: nvidia-lib
|
|
mountPath: /usr/local/nvidia/lib64
|
|
readOnly: true
|
|
volumes:
|
|
- name: dev
|
|
hostPath:
|
|
path: /dev
|
|
- name: nvidia-lib
|
|
hostPath:
|
|
path: /usr/local/nvidia/lib64
|
|
"""
|
|
|
|
stdin, stdout, stderr = c.exec_command("cat > /tmp/dcgm.yaml")
|
|
stdin.write(manifest)
|
|
stdin.close()
|
|
stdout.channel.recv_exit_status()
|
|
|
|
print("Applying DCGM Exporter with NVML mounts:")
|
|
print(run("kubectl apply -f /tmp/dcgm.yaml"))
|
|
time.sleep(10)
|
|
pod = run("kubectl get pod -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o jsonpath='{.items[0].metadata.name}'")
|
|
print("Logs from", pod)
|
|
print(run(f"kubectl logs -n ai-stack {pod} --tail=25"))
|
|
|
|
c.close()
|
|
|