import paramiko import sys import time sys.stdout.reconfigure(encoding='utf-8') MASTER_IP = '140.44.4.41' KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' TARGET_VERSION = 'v1.35.9+k3s1' ENCODED_VER = 'v1.35.9%2Bk3s1' masters = [ ('sninfr0lnxctl01', '140.44.4.41'), ('sninfr0lnxctl02', '140.44.4.42'), ('sninfr0lnxctl03', '140.44.4.43'), ('sninfr0lnxctl04', '140.44.4.44') ] workers = [ ('sninfr0lnxwrk01', '140.44.4.51'), ('sninfr0lnxwrk02', '140.44.4.52'), ('sninfr0lnxwrk03', '140.44.4.53'), ('sninfr0lnxwrk04', '140.44.4.54'), ('sninfr0lnxwrk05', '140.44.4.55'), ('sninfr0lnxwrk06', '140.44.4.56') ] master_ssh = paramiko.SSHClient() master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===") # 1. Save etcd snapshot print("[*] Saving pre-upgrade etcd snapshot...") stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-135-spaceport-{int(time.time())}') stdout.channel.recv_exit_status() print("[*] Etcd snapshot complete.") # 2. Upgrade Masters for name, ip in masters: print(f"\n=======================================================") print(f"[*] Upgrading Spaceport Master {name} ({ip})...") print(f"=======================================================") m_ssh = paramiko.SSHClient() m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) cmd = f""" set -e curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 chmod +x /usr/local/bin/k3s.new mv /usr/local/bin/k3s.new /usr/local/bin/k3s systemctl restart k3s """ stdin, stdout, stderr = m_ssh.exec_command(cmd) exit_code = stdout.channel.recv_exit_status() if exit_code != 0: print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") sys.exit(1) for _ in range(30): _, out, _ = m_ssh.exec_command('systemctl is-active k3s') if out.read().decode().strip() == 'active': print(f"[*] k3s service is active on {name}!") break time.sleep(3) m_ssh.close() for _ in range(35): _, out, _ = master_ssh.exec_command( f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" ) if out.read().decode().strip() == 'True': print(f"[OK] Master {name} is Ready!") break time.sleep(4) print("[*] Settling etcd consensus (15s)...") time.sleep(15) # 3. Upgrade Workers for name, ip in workers: print(f"\n=======================================================") print(f"[*] Upgrading Spaceport Worker {name} ({ip})...") print(f"=======================================================") print(f"[*] Cordoning {name}...") master_ssh.exec_command(f'kubectl cordon {name}') print(f"[*] Draining {name}...") stdin, stdout, stderr = master_ssh.exec_command( f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' ) stdout.channel.recv_exit_status() w_ssh = paramiko.SSHClient() w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) cmd = f""" set -e curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 chmod +x /usr/local/bin/k3s.new mv /usr/local/bin/k3s.new /usr/local/bin/k3s systemctl restart k3s-agent """ stdin, stdout, stderr = w_ssh.exec_command(cmd) exit_code = stdout.channel.recv_exit_status() if exit_code != 0: print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") sys.exit(1) for _ in range(30): _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') if out.read().decode().strip() == 'active': print(f"[*] k3s-agent is active on {name}!") break time.sleep(3) w_ssh.close() print(f"[*] Uncordoning {name}...") master_ssh.exec_command(f'kubectl uncordon {name}') for _ in range(35): _, out, _ = master_ssh.exec_command( f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" ) if out.read().decode().strip() == 'True': print(f"[OK] Worker {name} is Ready!") break time.sleep(4) print("[*] Settling Longhorn replicas (15s)...") time.sleep(15) print("\n=== FINAL SPACEPORT NODES ===") _, out, _ = master_ssh.exec_command('kubectl get nodes') print(out.read().decode()) master_ssh.close() print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!")