196 lines
6.7 KiB
Python
196 lines
6.7 KiB
Python
import paramiko
|
|
import sys
|
|
import time
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
MASTER_IP = '140.44.4.41'
|
|
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
|
|
|
|
workers = [
|
|
('sninfr0lnxwrk06', '140.44.4.56'),
|
|
('sninfr0lnxwrk05', '140.44.4.55'),
|
|
('sninfr0lnxwrk04', '140.44.4.54'),
|
|
('sninfr0lnxwrk03', '140.44.4.53'),
|
|
('sninfr0lnxwrk02', '140.44.4.52'),
|
|
('sninfr0lnxwrk01', '140.44.4.51')
|
|
]
|
|
|
|
masters = [
|
|
('sninfr0lnxctl04', '140.44.4.44'),
|
|
('sninfr0lnxctl03', '140.44.4.43'),
|
|
('sninfr0lnxctl02', '140.44.4.42'),
|
|
('sninfr0lnxctl01', '140.44.4.41')
|
|
]
|
|
|
|
upgrade_cmd = """
|
|
echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
|
|
echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get update -qq
|
|
apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y
|
|
apt-get autoremove -y
|
|
"""
|
|
|
|
master_ssh = paramiko.SSHClient()
|
|
master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
# Phase 1: Workers
|
|
for name, ip in workers:
|
|
print(f"\n=======================================================")
|
|
print(f"[*] Upgrading Spaceport worker {name} ({ip})...")
|
|
print(f"=======================================================")
|
|
|
|
# 1. Cordon
|
|
print(f"[*] Cordoning {name}...")
|
|
master_ssh.exec_command(f'kubectl cordon {name}')
|
|
|
|
# 2. Drain
|
|
print(f"[*] Draining {name} (ignoring daemonsets)...")
|
|
_, stdout, stderr = master_ssh.exec_command(
|
|
f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s'
|
|
)
|
|
print("Drain:", stdout.read().decode().strip()[:200])
|
|
|
|
# 3. Connect to worker
|
|
w_ssh = paramiko.SSHClient()
|
|
w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
print(f"[*] Running dist-upgrade on {name}...")
|
|
stdin, stdout, stderr = w_ssh.exec_command(upgrade_cmd)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
print(f"[*] Upgrade finished with exit code {exit_code}")
|
|
|
|
# Check reboot
|
|
_, out, _ = w_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
|
|
reboot_needed = out.read().decode().strip() == 'yes'
|
|
print(f"[*] Reboot required: {reboot_needed}")
|
|
|
|
if reboot_needed:
|
|
print(f"[*] Rebooting {name}...")
|
|
try:
|
|
w_ssh.exec_command("reboot")
|
|
except Exception:
|
|
pass
|
|
w_ssh.close()
|
|
|
|
print("[*] Waiting 30s for reboot...")
|
|
time.sleep(30)
|
|
|
|
reconnected = False
|
|
for attempt in range(25):
|
|
try:
|
|
w_ssh = paramiko.SSHClient()
|
|
w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6)
|
|
reconnected = True
|
|
print(f"[*] {name} reconnected successfully on attempt {attempt+1}!")
|
|
break
|
|
except Exception:
|
|
time.sleep(4)
|
|
|
|
if not reconnected:
|
|
print(f"[!] ERROR: {name} failed to reconnect!")
|
|
sys.exit(1)
|
|
|
|
_, out, _ = w_ssh.exec_command("systemctl is-active k3s-agent")
|
|
print(f" K3sAgent: {out.read().decode().strip()}")
|
|
w_ssh.close()
|
|
else:
|
|
w_ssh.close()
|
|
|
|
# 4. Uncordon
|
|
print(f"[*] Uncordoning {name}...")
|
|
master_ssh.exec_command(f'kubectl uncordon {name}')
|
|
|
|
# 5. Verify Ready
|
|
print(f"[*] Verifying node Ready state in Kubernetes...")
|
|
for _ in range(25):
|
|
_, out, _ = master_ssh.exec_command(
|
|
f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'"
|
|
)
|
|
if out.read().decode().strip() == 'True':
|
|
print(f"[OK] {name} is fully Ready!")
|
|
break
|
|
time.sleep(4)
|
|
|
|
# 6. Settle Longhorn
|
|
print(f"[*] Settling Longhorn storage replicas (15s)...")
|
|
time.sleep(15)
|
|
|
|
# Phase 2: Masters
|
|
for name, ip in masters:
|
|
print(f"\n=======================================================")
|
|
print(f"[*] Upgrading Spaceport master {name} ({ip})...")
|
|
print(f"=======================================================")
|
|
|
|
m_ssh = paramiko.SSHClient()
|
|
m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
print(f"[*] Running dist-upgrade on {name}...")
|
|
stdin, stdout, stderr = m_ssh.exec_command(upgrade_cmd)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
print(f"[*] Upgrade finished with exit code {exit_code}")
|
|
|
|
_, out, _ = m_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
|
|
reboot_needed = out.read().decode().strip() == 'yes'
|
|
print(f"[*] Reboot required: {reboot_needed}")
|
|
|
|
if reboot_needed:
|
|
print(f"[*] Rebooting {name}...")
|
|
try:
|
|
m_ssh.exec_command("reboot")
|
|
except Exception:
|
|
pass
|
|
m_ssh.close()
|
|
|
|
print("[*] Waiting 30s for reboot...")
|
|
time.sleep(30)
|
|
|
|
reconnected = False
|
|
for attempt in range(25):
|
|
try:
|
|
m_ssh = paramiko.SSHClient()
|
|
m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6)
|
|
reconnected = True
|
|
print(f"[*] {name} reconnected successfully on attempt {attempt+1}!")
|
|
break
|
|
except Exception:
|
|
time.sleep(4)
|
|
|
|
if not reconnected:
|
|
print(f"[!] ERROR: {name} failed to reconnect!")
|
|
sys.exit(1)
|
|
|
|
_, out, _ = m_ssh.exec_command("systemctl is-active k3s")
|
|
print(f" K3s: {out.read().decode().strip()}")
|
|
m_ssh.close()
|
|
else:
|
|
m_ssh.close()
|
|
|
|
print(f"[*] Settling etcd consensus (15s)...")
|
|
time.sleep(15)
|
|
|
|
master_ssh.close()
|
|
|
|
# Final Verification from 140.44.4.41
|
|
final_ssh = paramiko.SSHClient()
|
|
final_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
final_ssh.connect('140.44.4.41', username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
print("\n=== FINAL SPACEPORT NODES ===")
|
|
_, out, _ = final_ssh.exec_command('kubectl get nodes')
|
|
print(out.read().decode())
|
|
|
|
print("=== FINAL LONGHORN VOLUMES ===")
|
|
_, out, _ = final_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system')
|
|
print(out.read().decode())
|
|
|
|
final_ssh.close()
|
|
print("[SUCCESS] Spaceport rolling OS upgrade completed flawlessly!")
|
|
|