import paramiko import sys import time sys.stdout.reconfigure(encoding='utf-8') MASTER_IP = '140.44.4.41' KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' workers = [ ('sninfr0lnxwrk06', '140.44.4.56'), ('sninfr0lnxwrk05', '140.44.4.55'), ('sninfr0lnxwrk04', '140.44.4.54'), ('sninfr0lnxwrk03', '140.44.4.53'), ('sninfr0lnxwrk02', '140.44.4.52'), ('sninfr0lnxwrk01', '140.44.4.51') ] masters = [ ('sninfr0lnxctl04', '140.44.4.44'), ('sninfr0lnxctl03', '140.44.4.43'), ('sninfr0lnxctl02', '140.44.4.42'), ('sninfr0lnxctl01', '140.44.4.41') ] upgrade_cmd = """ echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y apt-get autoremove -y """ master_ssh = paramiko.SSHClient() master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) # Phase 1: Workers for name, ip in workers: print(f"\n=======================================================") print(f"[*] Upgrading Spaceport worker {name} ({ip})...") print(f"=======================================================") # 1. Cordon print(f"[*] Cordoning {name}...") master_ssh.exec_command(f'kubectl cordon {name}') # 2. Drain print(f"[*] Draining {name} (ignoring daemonsets)...") _, stdout, stderr = master_ssh.exec_command( f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' ) print("Drain:", stdout.read().decode().strip()[:200]) # 3. Connect to worker w_ssh = paramiko.SSHClient() w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) print(f"[*] Running dist-upgrade on {name}...") stdin, stdout, stderr = w_ssh.exec_command(upgrade_cmd) exit_code = stdout.channel.recv_exit_status() print(f"[*] Upgrade finished with exit code {exit_code}") # Check reboot _, out, _ = w_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") reboot_needed = out.read().decode().strip() == 'yes' print(f"[*] Reboot required: {reboot_needed}") if reboot_needed: print(f"[*] Rebooting {name}...") try: w_ssh.exec_command("reboot") except Exception: pass w_ssh.close() print("[*] Waiting 30s for reboot...") time.sleep(30) reconnected = False for attempt in range(25): try: w_ssh = paramiko.SSHClient() w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6) reconnected = True print(f"[*] {name} reconnected successfully on attempt {attempt+1}!") break except Exception: time.sleep(4) if not reconnected: print(f"[!] ERROR: {name} failed to reconnect!") sys.exit(1) _, out, _ = w_ssh.exec_command("systemctl is-active k3s-agent") print(f" K3sAgent: {out.read().decode().strip()}") w_ssh.close() else: w_ssh.close() # 4. Uncordon print(f"[*] Uncordoning {name}...") master_ssh.exec_command(f'kubectl uncordon {name}') # 5. Verify Ready print(f"[*] Verifying node Ready state in Kubernetes...") for _ in range(25): _, out, _ = master_ssh.exec_command( f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" ) if out.read().decode().strip() == 'True': print(f"[OK] {name} is fully Ready!") break time.sleep(4) # 6. Settle Longhorn print(f"[*] Settling Longhorn storage replicas (15s)...") time.sleep(15) # Phase 2: Masters for name, ip in masters: print(f"\n=======================================================") print(f"[*] Upgrading Spaceport master {name} ({ip})...") print(f"=======================================================") m_ssh = paramiko.SSHClient() m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) print(f"[*] Running dist-upgrade on {name}...") stdin, stdout, stderr = m_ssh.exec_command(upgrade_cmd) exit_code = stdout.channel.recv_exit_status() print(f"[*] Upgrade finished with exit code {exit_code}") _, out, _ = m_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") reboot_needed = out.read().decode().strip() == 'yes' print(f"[*] Reboot required: {reboot_needed}") if reboot_needed: print(f"[*] Rebooting {name}...") try: m_ssh.exec_command("reboot") except Exception: pass m_ssh.close() print("[*] Waiting 30s for reboot...") time.sleep(30) reconnected = False for attempt in range(25): try: m_ssh = paramiko.SSHClient() m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6) reconnected = True print(f"[*] {name} reconnected successfully on attempt {attempt+1}!") break except Exception: time.sleep(4) if not reconnected: print(f"[!] ERROR: {name} failed to reconnect!") sys.exit(1) _, out, _ = m_ssh.exec_command("systemctl is-active k3s") print(f" K3s: {out.read().decode().strip()}") m_ssh.close() else: m_ssh.close() print(f"[*] Settling etcd consensus (15s)...") time.sleep(15) master_ssh.close() # Final Verification from 140.44.4.41 final_ssh = paramiko.SSHClient() final_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) final_ssh.connect('140.44.4.41', username='root', key_filename=KEY_FILE, timeout=10) print("\n=== FINAL SPACEPORT NODES ===") _, out, _ = final_ssh.exec_command('kubectl get nodes') print(out.read().decode()) print("=== FINAL LONGHORN VOLUMES ===") _, out, _ = final_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system') print(out.read().decode()) final_ssh.close() print("[SUCCESS] Spaceport rolling OS upgrade completed flawlessly!")