import paramiko import time SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" c = paramiko.SSHClient() c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10) def k8s(cmd): full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'" _, o, e = c.exec_command(full_cmd) return o.read().decode().strip(), e.read().decode().strip() # Try clearing failedAt k8s("echo eyJzcGVjIjp7ImZhaWxlZEF0IjoiIn19 | base64 -d > /tmp/clear_failed.json") out, _ = k8s("cat /tmp/clear_failed.json") print("Patch file:", out) out, err = k8s("kubectl patch replica.longhorn.io pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495-r-3a9069af -n longhorn-system --type=merge --patch-file=/tmp/clear_failed.json") print("Patch failedAt to empty:", out, err) # Check replica spec now out, _ = k8s("kubectl get replica.longhorn.io pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495-r-3a9069af -n longhorn-system -o yaml | grep -E 'failedAt|healthyAt|salvageRequested'") print("\nReplica spec after patch:") print(out) # Check longhorn manager log for this volume time.sleep(3) out, _ = k8s("kubectl logs -n longhorn-system -l app=longhorn-manager --tail=20 | grep pvc-4da99c48") print("\nLonghorn manager log:") print(out) c.close()