import paramiko import time import json SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" c = paramiko.SSHClient() c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10) def k8s(cmd): full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'" _, o, e = c.exec_command(full_cmd) return o.read().decode().strip(), e.read().decode().strip() # 1. Write the salvage patch file safely using base64 k8s("echo eyJzcGVjIjp7InNhbHZhZ2VSZXF1ZXN0ZWQiOnRydWV9fQ== | base64 -d > /tmp/salvage_patch.json") out, _ = k8s("cat /tmp/salvage_patch.json") print("Patch file content:", out) # 2. Get all faulted volumes cmd = "kubectl get volumes.longhorn.io -n longhorn-system -o json" out, _ = k8s(cmd) vol_data = json.loads(out) faulted_vols = [] for item in vol_data.get("items", []): if item.get("status", {}).get("robustness") == "faulted": faulted_vols.append(item["metadata"]["name"]) print(f"Faulted volumes count: {len(faulted_vols)} -> {faulted_vols}") # 3. For each faulted volume, find its replicas and apply salvageRequested=true for vol in faulted_vols: print(f"\nProcessing volume: {vol}") # Get replicas for this volume r_cmd = f"kubectl get replicas.longhorn.io -n longhorn-system -l longhornvolume={vol} -o jsonpath='{{.items[*].metadata.name}}'" reps_out, _ = k8s(r_cmd) replicas = reps_out.split() print(f" Replicas ({len(replicas)}): {replicas}") for rep in replicas: p_cmd = f"kubectl patch replica.longhorn.io {rep} -n longhorn-system --type=merge --patch-file=/tmp/salvage_patch.json" p_out, p_err = k8s(p_cmd) print(f" Patched {rep}: {p_out}") print("\nWaiting 10 seconds for Longhorn volume reconciler...") time.sleep(10) # Check volume statuses out, _ = k8s("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness,REPLICAS:.status.currentReplicas") print("\nLonghorn Volumes Status:") print(out) # Check pods in ai-stack out, _ = k8s("kubectl get pods -n ai-stack -o wide") print("\nPods in ai-stack:") print(out) c.close()