149 lines
5.0 KiB
Python
149 lines
5.0 KiB
Python
import paramiko
|
|
import sys
|
|
import time
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
MASTER_IP = '140.44.4.11'
|
|
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
|
|
TARGET_VERSION = 'v1.36.5+k3s1'
|
|
ENCODED_VER = 'v1.36.5%2Bk3s1'
|
|
|
|
masters = [
|
|
('sninfr2lnxctl01', '140.44.4.11'),
|
|
('sninfr2lnxctl02', '140.44.4.12'),
|
|
('sninfr2lnxctl03', '140.44.4.13')
|
|
]
|
|
|
|
workers = [
|
|
('sninfr2lnxwrk01', '140.44.4.21'),
|
|
('sninfr2lnxwrk02', '140.44.4.22'),
|
|
('sninfr2lnxwrk03', '140.44.4.23'),
|
|
('sninfr2lnxwrk04', '140.44.4.24'),
|
|
('sninfr2lnxwrk05', '140.44.4.25'),
|
|
('sninfr2lnxwrk06', '140.44.4.26'),
|
|
('sninfr2lnxwrk07', '140.44.4.27')
|
|
]
|
|
|
|
master_ssh = paramiko.SSHClient()
|
|
master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
print(f"=== INITIATING AERODROME UPGRADE TO {TARGET_VERSION} ===")
|
|
|
|
# 1. Save etcd snapshot
|
|
print("[*] Saving pre-upgrade etcd snapshot...")
|
|
stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-136-aerodrome-{int(time.time())}')
|
|
stdout.channel.recv_exit_status()
|
|
print("[*] Etcd snapshot complete.")
|
|
|
|
# 2. Upgrade Masters
|
|
for name, ip in masters:
|
|
print(f"\n=======================================================")
|
|
print(f"[*] Upgrading Aerodrome Master {name} ({ip})...")
|
|
print(f"=======================================================")
|
|
|
|
m_ssh = paramiko.SSHClient()
|
|
m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
cmd = f"""
|
|
set -e
|
|
curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
|
|
test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000
|
|
chmod +x /usr/local/bin/k3s.new
|
|
mv /usr/local/bin/k3s.new /usr/local/bin/k3s
|
|
systemctl restart k3s
|
|
"""
|
|
stdin, stdout, stderr = m_ssh.exec_command(cmd)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
if exit_code != 0:
|
|
print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}")
|
|
sys.exit(1)
|
|
|
|
for _ in range(30):
|
|
_, out, _ = m_ssh.exec_command('systemctl is-active k3s')
|
|
if out.read().decode().strip() == 'active':
|
|
print(f"[*] k3s service is active on {name}!")
|
|
break
|
|
time.sleep(3)
|
|
|
|
m_ssh.close()
|
|
|
|
for _ in range(35):
|
|
_, out, _ = master_ssh.exec_command(
|
|
f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'"
|
|
)
|
|
if out.read().decode().strip() == 'True':
|
|
print(f"[OK] Master {name} is Ready!")
|
|
break
|
|
time.sleep(4)
|
|
|
|
print("[*] Settling etcd consensus (15s)...")
|
|
time.sleep(15)
|
|
|
|
# 3. Upgrade Workers
|
|
for name, ip in workers:
|
|
print(f"\n=======================================================")
|
|
print(f"[*] Upgrading Aerodrome Worker {name} ({ip})...")
|
|
print(f"=======================================================")
|
|
|
|
print(f"[*] Cordoning {name}...")
|
|
master_ssh.exec_command(f'kubectl cordon {name}')
|
|
|
|
print(f"[*] Draining {name}...")
|
|
stdin, stdout, stderr = master_ssh.exec_command(
|
|
f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s'
|
|
)
|
|
stdout.channel.recv_exit_status()
|
|
|
|
w_ssh = paramiko.SSHClient()
|
|
w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
|
|
|
|
cmd = f"""
|
|
set -e
|
|
curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
|
|
test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000
|
|
chmod +x /usr/local/bin/k3s.new
|
|
mv /usr/local/bin/k3s.new /usr/local/bin/k3s
|
|
systemctl restart k3s-agent
|
|
"""
|
|
stdin, stdout, stderr = w_ssh.exec_command(cmd)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
if exit_code != 0:
|
|
print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}")
|
|
sys.exit(1)
|
|
|
|
for _ in range(30):
|
|
_, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent')
|
|
if out.read().decode().strip() == 'active':
|
|
print(f"[*] k3s-agent is active on {name}!")
|
|
break
|
|
time.sleep(3)
|
|
|
|
w_ssh.close()
|
|
|
|
print(f"[*] Uncordoning {name}...")
|
|
master_ssh.exec_command(f'kubectl uncordon {name}')
|
|
|
|
for _ in range(35):
|
|
_, out, _ = master_ssh.exec_command(
|
|
f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'"
|
|
)
|
|
if out.read().decode().strip() == 'True':
|
|
print(f"[OK] Worker {name} is Ready!")
|
|
break
|
|
time.sleep(4)
|
|
|
|
print("[*] Settling Longhorn replicas (15s)...")
|
|
time.sleep(15)
|
|
|
|
print("\n=== FINAL AERODROME NODES ===")
|
|
_, out, _ = master_ssh.exec_command('kubectl get nodes')
|
|
print(out.read().decode())
|
|
|
|
master_ssh.close()
|
|
print(f"\n[SUCCESS] AERODROME FULLY UPGRADED TO {TARGET_VERSION}!")
|
|
|