feat: add various infrastructure check and automation scripts
This commit is contained in:
84
scripts/audit_spacedock_failure.py
Normal file
84
scripts/audit_spacedock_failure.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
HARVESTER_VIP = "140.44.4.90"
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
try:
|
||||
print(f"Connecting to Harvester VIP {HARVESTER_VIP}...")
|
||||
c.connect(HARVESTER_VIP, username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
print("Connected to Harvester VIP!")
|
||||
|
||||
def run_k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
stdin, stdout, stderr = c.exec_command(full_cmd)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
return out
|
||||
|
||||
print("\n=== 1. HARVESTER CLUSTER NODES ===")
|
||||
print(run_k8s("kubectl get nodes -o wide"))
|
||||
|
||||
print("\n=== 2. NODE UPTIME ===")
|
||||
print(run_k8s("uptime"))
|
||||
|
||||
print("\n=== 3. VIRTUAL MACHINES (KubeVirt) ===")
|
||||
print(run_k8s("kubectl get vms,vmis -A -o wide"))
|
||||
|
||||
print("\n=== 4. HARVESTER SYSTEM POD STATUS (Not Running / Degraded) ===")
|
||||
print(run_k8s("kubectl get pods -A | grep -v 'Running\\|Completed'"))
|
||||
|
||||
print("\n=== 5. LONGHORN STORAGE VOLUME STATUS ===")
|
||||
print(run_k8s("kubectl get volumes.longhorn.io -n longhorn-system"))
|
||||
|
||||
c.close()
|
||||
except Exception as e:
|
||||
print(f"Error connecting to Harvester: {e}")
|
||||
|
||||
# Check iDRACs
|
||||
idracs = {
|
||||
"Spacedock Node 1": "140.44.44.121",
|
||||
"Spacedock Node 2": "140.44.44.122",
|
||||
"Spacedock Node 3": "140.44.44.123",
|
||||
"Spacedock Node 4": "140.44.44.124",
|
||||
}
|
||||
|
||||
print("\n=== 6. IDRAC SYSTEM EVENT LOGS & POWER STATUS ===")
|
||||
for name, ip in idracs.items():
|
||||
print(f"\n--- {name} ({ip}) ---")
|
||||
try:
|
||||
ic = paramiko.SSHClient()
|
||||
ic.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ic.connect(ip, username="root", password="Katzir476!", timeout=8)
|
||||
chan = ic.invoke_shell()
|
||||
time.sleep(1.5)
|
||||
if chan.recv_ready():
|
||||
chan.recv(4096)
|
||||
|
||||
def racadm(cmd):
|
||||
chan.send(cmd + "\r\n")
|
||||
time.sleep(2)
|
||||
out = ""
|
||||
while chan.recv_ready():
|
||||
out += chan.recv(8192).decode('utf-8', errors='ignore')
|
||||
time.sleep(0.5)
|
||||
return out
|
||||
|
||||
pwr = racadm("racadm serveraction powerstatus")
|
||||
for line in pwr.splitlines():
|
||||
if "power" in line.lower() or "server" in line.lower():
|
||||
print(line.strip())
|
||||
|
||||
sel = racadm("racadm getsel -s 10")
|
||||
print("Last 10 SEL Events:")
|
||||
for line in sel.splitlines()[-12:]:
|
||||
print(" ", line.strip())
|
||||
|
||||
ic.close()
|
||||
except Exception as e:
|
||||
print(f"Error querying {name}: {e}")
|
||||
|
||||
Reference in New Issue
Block a user