60 lines
1.6 KiB
Python
60 lines
1.6 KiB
Python
import paramiko
|
|
|
|
c = paramiko.SSHClient()
|
|
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
|
|
|
def run(cmd):
|
|
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
|
|
out = stdout.read().decode().strip()
|
|
err = stderr.read().decode().strip()
|
|
return out + ("\nERR: " + err if err else "")
|
|
|
|
manifest = """
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: dcgm-exporter
|
|
namespace: ai-stack
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
spec:
|
|
hostNetwork: true
|
|
containers:
|
|
- name: dcgm-exporter
|
|
image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
|
|
env:
|
|
- name: DCGM_EXPORTER_LISTEN
|
|
value: ":9400"
|
|
- name: DCGM_EXPORTER_KUBERNETES
|
|
value: "false"
|
|
securityContext:
|
|
privileged: true
|
|
ports:
|
|
- name: metrics
|
|
containerPort: 9400
|
|
"""
|
|
|
|
stdin, stdout, stderr = c.exec_command("cat > /tmp/dcgm.yaml")
|
|
stdin.write(manifest)
|
|
stdin.close()
|
|
stdout.channel.recv_exit_status()
|
|
|
|
print("Applying DCGM Exporter:")
|
|
print(run("kubectl apply -f /tmp/dcgm.yaml"))
|
|
import time
|
|
time.sleep(5)
|
|
print("Pods:")
|
|
print(run("kubectl get pods -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o wide"))
|
|
|
|
c.close()
|
|
|