feat: add various infrastructure check and automation scripts

This commit is contained in:
2026-11-15 17:17:41 -05:00
parent 336b7759d0
commit b301aa3ece
232 changed files with 8274 additions and 298 deletions

View File

@@ -0,0 +1,59 @@
import paramiko
import time
import json
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
def k8s(cmd):
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
_, o, e = c.exec_command(full_cmd)
return o.read().decode().strip(), e.read().decode().strip()
# 1. Write the salvage patch file safely using base64
k8s("echo eyJzcGVjIjp7InNhbHZhZ2VSZXF1ZXN0ZWQiOnRydWV9fQ== | base64 -d > /tmp/salvage_patch.json")
out, _ = k8s("cat /tmp/salvage_patch.json")
print("Patch file content:", out)
# 2. Get all faulted volumes
cmd = "kubectl get volumes.longhorn.io -n longhorn-system -o json"
out, _ = k8s(cmd)
vol_data = json.loads(out)
faulted_vols = []
for item in vol_data.get("items", []):
if item.get("status", {}).get("robustness") == "faulted":
faulted_vols.append(item["metadata"]["name"])
print(f"Faulted volumes count: {len(faulted_vols)} -> {faulted_vols}")
# 3. For each faulted volume, find its replicas and apply salvageRequested=true
for vol in faulted_vols:
print(f"\nProcessing volume: {vol}")
# Get replicas for this volume
r_cmd = f"kubectl get replicas.longhorn.io -n longhorn-system -l longhornvolume={vol} -o jsonpath='{{.items[*].metadata.name}}'"
reps_out, _ = k8s(r_cmd)
replicas = reps_out.split()
print(f" Replicas ({len(replicas)}): {replicas}")
for rep in replicas:
p_cmd = f"kubectl patch replica.longhorn.io {rep} -n longhorn-system --type=merge --patch-file=/tmp/salvage_patch.json"
p_out, p_err = k8s(p_cmd)
print(f" Patched {rep}: {p_out}")
print("\nWaiting 10 seconds for Longhorn volume reconciler...")
time.sleep(10)
# Check volume statuses
out, _ = k8s("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness,REPLICAS:.status.currentReplicas")
print("\nLonghorn Volumes Status:")
print(out)
# Check pods in ai-stack
out, _ = k8s("kubectl get pods -n ai-stack -o wide")
print("\nPods in ai-stack:")
print(out)
c.close()