feat: add various infrastructure check and automation scripts
This commit is contained in:
59
scripts/salvage_all_volumes.py
Normal file
59
scripts/salvage_all_volumes.py
Normal file
@@ -0,0 +1,59 @@
|
||||
import paramiko
|
||||
import time
|
||||
import json
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
# 1. Write the salvage patch file safely using base64
|
||||
k8s("echo eyJzcGVjIjp7InNhbHZhZ2VSZXF1ZXN0ZWQiOnRydWV9fQ== | base64 -d > /tmp/salvage_patch.json")
|
||||
out, _ = k8s("cat /tmp/salvage_patch.json")
|
||||
print("Patch file content:", out)
|
||||
|
||||
# 2. Get all faulted volumes
|
||||
cmd = "kubectl get volumes.longhorn.io -n longhorn-system -o json"
|
||||
out, _ = k8s(cmd)
|
||||
vol_data = json.loads(out)
|
||||
|
||||
faulted_vols = []
|
||||
for item in vol_data.get("items", []):
|
||||
if item.get("status", {}).get("robustness") == "faulted":
|
||||
faulted_vols.append(item["metadata"]["name"])
|
||||
|
||||
print(f"Faulted volumes count: {len(faulted_vols)} -> {faulted_vols}")
|
||||
|
||||
# 3. For each faulted volume, find its replicas and apply salvageRequested=true
|
||||
for vol in faulted_vols:
|
||||
print(f"\nProcessing volume: {vol}")
|
||||
# Get replicas for this volume
|
||||
r_cmd = f"kubectl get replicas.longhorn.io -n longhorn-system -l longhornvolume={vol} -o jsonpath='{{.items[*].metadata.name}}'"
|
||||
reps_out, _ = k8s(r_cmd)
|
||||
replicas = reps_out.split()
|
||||
print(f" Replicas ({len(replicas)}): {replicas}")
|
||||
for rep in replicas:
|
||||
p_cmd = f"kubectl patch replica.longhorn.io {rep} -n longhorn-system --type=merge --patch-file=/tmp/salvage_patch.json"
|
||||
p_out, p_err = k8s(p_cmd)
|
||||
print(f" Patched {rep}: {p_out}")
|
||||
|
||||
print("\nWaiting 10 seconds for Longhorn volume reconciler...")
|
||||
time.sleep(10)
|
||||
|
||||
# Check volume statuses
|
||||
out, _ = k8s("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness,REPLICAS:.status.currentReplicas")
|
||||
print("\nLonghorn Volumes Status:")
|
||||
print(out)
|
||||
|
||||
# Check pods in ai-stack
|
||||
out, _ = k8s("kubectl get pods -n ai-stack -o wide")
|
||||
print("\nPods in ai-stack:")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
Reference in New Issue
Block a user