Files
SnarfCode/scripts/audit_spacedock_failure.py

85 lines
2.7 KiB
Python

import paramiko
import json
import time
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
HARVESTER_VIP = "140.44.4.90"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
print(f"Connecting to Harvester VIP {HARVESTER_VIP}...")
c.connect(HARVESTER_VIP, username="rancher", key_filename=SSH_KEY, timeout=10)
print("Connected to Harvester VIP!")
def run_k8s(cmd):
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
stdin, stdout, stderr = c.exec_command(full_cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
return out
print("\n=== 1. HARVESTER CLUSTER NODES ===")
print(run_k8s("kubectl get nodes -o wide"))
print("\n=== 2. NODE UPTIME ===")
print(run_k8s("uptime"))
print("\n=== 3. VIRTUAL MACHINES (KubeVirt) ===")
print(run_k8s("kubectl get vms,vmis -A -o wide"))
print("\n=== 4. HARVESTER SYSTEM POD STATUS (Not Running / Degraded) ===")
print(run_k8s("kubectl get pods -A | grep -v 'Running\\|Completed'"))
print("\n=== 5. LONGHORN STORAGE VOLUME STATUS ===")
print(run_k8s("kubectl get volumes.longhorn.io -n longhorn-system"))
c.close()
except Exception as e:
print(f"Error connecting to Harvester: {e}")
# Check iDRACs
idracs = {
"Spacedock Node 1": "140.44.44.121",
"Spacedock Node 2": "140.44.44.122",
"Spacedock Node 3": "140.44.44.123",
"Spacedock Node 4": "140.44.44.124",
}
print("\n=== 6. IDRAC SYSTEM EVENT LOGS & POWER STATUS ===")
for name, ip in idracs.items():
print(f"\n--- {name} ({ip}) ---")
try:
ic = paramiko.SSHClient()
ic.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ic.connect(ip, username="root", password="Katzir476!", timeout=8)
chan = ic.invoke_shell()
time.sleep(1.5)
if chan.recv_ready():
chan.recv(4096)
def racadm(cmd):
chan.send(cmd + "\r\n")
time.sleep(2)
out = ""
while chan.recv_ready():
out += chan.recv(8192).decode('utf-8', errors='ignore')
time.sleep(0.5)
return out
pwr = racadm("racadm serveraction powerstatus")
for line in pwr.splitlines():
if "power" in line.lower() or "server" in line.lower():
print(line.strip())
sel = racadm("racadm getsel -s 10")
print("Last 10 SEL Events:")
for line in sel.splitlines()[-12:]:
print(" ", line.strip())
ic.close()
except Exception as e:
print(f"Error querying {name}: {e}")