Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add cluster auditing, management, and upgrade scripts

Browse Source
This commit is contained in:
Snarf
2026-11-04 17:58:07 -05:00
parent 8117326b14
commit 1b378b29f3
128 changed files with 4744 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
scripts/upgrade_spaceport_os.py Normal file
View File

@@ -0,0 +1,195 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
MASTER_IP = '140.44.4.41'
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
workers = [
('sninfr0lnxwrk06', '140.44.4.56'),
('sninfr0lnxwrk05', '140.44.4.55'),
('sninfr0lnxwrk04', '140.44.4.54'),
('sninfr0lnxwrk03', '140.44.4.53'),
('sninfr0lnxwrk02', '140.44.4.52'),
('sninfr0lnxwrk01', '140.44.4.51')
]
masters = [
('sninfr0lnxctl04', '140.44.4.44'),
('sninfr0lnxctl03', '140.44.4.43'),
('sninfr0lnxctl02', '140.44.4.42'),
('sninfr0lnxctl01', '140.44.4.41')
]
upgrade_cmd = """
echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y
apt-get autoremove -y
"""
master_ssh = paramiko.SSHClient()
master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
# Phase 1: Workers
for name, ip in workers:
print(f"\n=======================================================")
print(f"[*] Upgrading Spaceport worker {name} ({ip})...")
print(f"=======================================================")
# 1. Cordon
print(f"[*] Cordoning {name}...")
master_ssh.exec_command(f'kubectl cordon {name}')
# 2. Drain
print(f"[*] Draining {name} (ignoring daemonsets)...")
_, stdout, stderr = master_ssh.exec_command(
f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s'
)
print("Drain:", stdout.read().decode().strip()[:200])
# 3. Connect to worker
w_ssh = paramiko.SSHClient()
w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
print(f"[*] Running dist-upgrade on {name}...")
stdin, stdout, stderr = w_ssh.exec_command(upgrade_cmd)
exit_code = stdout.channel.recv_exit_status()
print(f"[*] Upgrade finished with exit code {exit_code}")
# Check reboot
_, out, _ = w_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
reboot_needed = out.read().decode().strip() == 'yes'
print(f"[*] Reboot required: {reboot_needed}")
if reboot_needed:
print(f"[*] Rebooting {name}...")
try:
w_ssh.exec_command("reboot")
except Exception:
pass
w_ssh.close()
print("[*] Waiting 30s for reboot...")
time.sleep(30)
reconnected = False
for attempt in range(25):
try:
w_ssh = paramiko.SSHClient()
w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6)
reconnected = True
print(f"[*] {name} reconnected successfully on attempt {attempt+1}!")
break
except Exception:
time.sleep(4)
if not reconnected:
print(f"[!] ERROR: {name} failed to reconnect!")
sys.exit(1)
_, out, _ = w_ssh.exec_command("systemctl is-active k3s-agent")
print(f" K3sAgent: {out.read().decode().strip()}")
w_ssh.close()
else:
w_ssh.close()
# 4. Uncordon
print(f"[*] Uncordoning {name}...")
master_ssh.exec_command(f'kubectl uncordon {name}')
# 5. Verify Ready
print(f"[*] Verifying node Ready state in Kubernetes...")
for _ in range(25):
_, out, _ = master_ssh.exec_command(
f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'"
)
if out.read().decode().strip() == 'True':
print(f"[OK] {name} is fully Ready!")
break
time.sleep(4)
# 6. Settle Longhorn
print(f"[*] Settling Longhorn storage replicas (15s)...")
time.sleep(15)
# Phase 2: Masters
for name, ip in masters:
print(f"\n=======================================================")
print(f"[*] Upgrading Spaceport master {name} ({ip})...")
print(f"=======================================================")
m_ssh = paramiko.SSHClient()
m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
print(f"[*] Running dist-upgrade on {name}...")
stdin, stdout, stderr = m_ssh.exec_command(upgrade_cmd)
exit_code = stdout.channel.recv_exit_status()
print(f"[*] Upgrade finished with exit code {exit_code}")
_, out, _ = m_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
reboot_needed = out.read().decode().strip() == 'yes'
print(f"[*] Reboot required: {reboot_needed}")
if reboot_needed:
print(f"[*] Rebooting {name}...")
try:
m_ssh.exec_command("reboot")
except Exception:
pass
m_ssh.close()
print("[*] Waiting 30s for reboot...")
time.sleep(30)
reconnected = False
for attempt in range(25):
try:
m_ssh = paramiko.SSHClient()
m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6)
reconnected = True
print(f"[*] {name} reconnected successfully on attempt {attempt+1}!")
break
except Exception:
time.sleep(4)
if not reconnected:
print(f"[!] ERROR: {name} failed to reconnect!")
sys.exit(1)
_, out, _ = m_ssh.exec_command("systemctl is-active k3s")
print(f" K3s: {out.read().decode().strip()}")
m_ssh.close()
else:
m_ssh.close()
print(f"[*] Settling etcd consensus (15s)...")
time.sleep(15)
master_ssh.close()
# Final Verification from 140.44.4.41
final_ssh = paramiko.SSHClient()
final_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
final_ssh.connect('140.44.4.41', username='root', key_filename=KEY_FILE, timeout=10)
print("\n=== FINAL SPACEPORT NODES ===")
_, out, _ = final_ssh.exec_command('kubectl get nodes')
print(out.read().decode())
print("=== FINAL LONGHORN VOLUMES ===")
_, out, _ = final_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system')
print(out.read().decode())
final_ssh.close()
print("[SUCCESS] Spaceport rolling OS upgrade completed flawlessly!")