Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add cluster auditing, management, and upgrade scripts

Browse Source
This commit is contained in:
Snarf
2026-11-04 17:58:07 -05:00
parent 8117326b14
commit 1b378b29f3
128 changed files with 4744 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
scripts/upgrade_spaceport_133.py Normal file
View File

@@ -0,0 +1,148 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
MASTER_IP = '140.44.4.41'
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
TARGET_VERSION = 'v1.33.13+k3s2'
ENCODED_VER = 'v1.33.13%2Bk3s2'
masters = [
('sninfr0lnxctl01', '140.44.4.41'),
('sninfr0lnxctl02', '140.44.4.42'),
('sninfr0lnxctl03', '140.44.4.43'),
('sninfr0lnxctl04', '140.44.4.44')
]
workers = [
('sninfr0lnxwrk01', '140.44.4.51'),
('sninfr0lnxwrk02', '140.44.4.52'),
('sninfr0lnxwrk03', '140.44.4.53'),
('sninfr0lnxwrk04', '140.44.4.54'),
('sninfr0lnxwrk05', '140.44.4.55'),
('sninfr0lnxwrk06', '140.44.4.56')
]
master_ssh = paramiko.SSHClient()
master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===")
# 1. Save etcd snapshot
print("[*] Saving pre-upgrade etcd snapshot...")
stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-133-spaceport-{int(time.time())}')
stdout.channel.recv_exit_status()
print("[*] Etcd snapshot complete.")
# 2. Upgrade Masters
for name, ip in masters:
print(f"\n=======================================================")
print(f"[*] Upgrading Spaceport Master {name} ({ip})...")
print(f"=======================================================")
m_ssh = paramiko.SSHClient()
m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
cmd = f"""
set -e
curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000
chmod +x /usr/local/bin/k3s.new
mv /usr/local/bin/k3s.new /usr/local/bin/k3s
systemctl restart k3s
"""
stdin, stdout, stderr = m_ssh.exec_command(cmd)
exit_code = stdout.channel.recv_exit_status()
if exit_code != 0:
print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}")
sys.exit(1)
for _ in range(30):
_, out, _ = m_ssh.exec_command('systemctl is-active k3s')
if out.read().decode().strip() == 'active':
print(f"[*] k3s service is active on {name}!")
break
time.sleep(3)
m_ssh.close()
for _ in range(35):
_, out, _ = master_ssh.exec_command(
f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'"
)
if out.read().decode().strip() == 'True':
print(f"[OK] Master {name} is Ready!")
break
time.sleep(4)
print("[*] Settling etcd consensus (15s)...")
time.sleep(15)
# 3. Upgrade Workers
for name, ip in workers:
print(f"\n=======================================================")
print(f"[*] Upgrading Spaceport Worker {name} ({ip})...")
print(f"=======================================================")
print(f"[*] Cordoning {name}...")
master_ssh.exec_command(f'kubectl cordon {name}')
print(f"[*] Draining {name}...")
stdin, stdout, stderr = master_ssh.exec_command(
f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s'
)
stdout.channel.recv_exit_status()
w_ssh = paramiko.SSHClient()
w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10)
cmd = f"""
set -e
curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000
chmod +x /usr/local/bin/k3s.new
mv /usr/local/bin/k3s.new /usr/local/bin/k3s
systemctl restart k3s-agent
"""
stdin, stdout, stderr = w_ssh.exec_command(cmd)
exit_code = stdout.channel.recv_exit_status()
if exit_code != 0:
print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}")
sys.exit(1)
for _ in range(30):
_, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent')
if out.read().decode().strip() == 'active':
print(f"[*] k3s-agent is active on {name}!")
break
time.sleep(3)
w_ssh.close()
print(f"[*] Uncordoning {name}...")
master_ssh.exec_command(f'kubectl uncordon {name}')
for _ in range(35):
_, out, _ = master_ssh.exec_command(
f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'"
)
if out.read().decode().strip() == 'True':
print(f"[OK] Worker {name} is Ready!")
break
time.sleep(4)
print("[*] Settling Longhorn replicas (15s)...")
time.sleep(15)
print("\n=== FINAL SPACEPORT NODES ===")
_, out, _ = master_ssh.exec_command('kubectl get nodes')
print(out.read().decode())
master_ssh.close()
print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!")