From 5b955fcbc3746a6901cdc1934bf353a4f907c559 Mon Sep 17 00:00:00 2001 From: Snarf Date: Thu, 5 Nov 2026 06:50:35 -0500 Subject: [PATCH] feat: add housekeeping and cleanup scripts for cluster maintenance --- scripts/check_housekeeping.py | 14 ++++++ scripts/check_spaceport_net.py | 26 ++++++++++++ scripts/cleanup_spaceport.py | 35 +++++++++++++++ scripts/post_upgrade_cleanup.py | 75 +++++++++++++++++++++++++++++++++ scripts/test_direct_41.py | 18 ++++++++ scripts/test_spaceport_snap.py | 19 +++++++++ scripts/test_ssh_spaceport.py | 18 ++++++++ 7 files changed, 205 insertions(+) create mode 100644 scripts/check_housekeeping.py create mode 100644 scripts/check_spaceport_net.py create mode 100644 scripts/cleanup_spaceport.py create mode 100644 scripts/post_upgrade_cleanup.py create mode 100644 scripts/test_direct_41.py create mode 100644 scripts/test_spaceport_snap.py create mode 100644 scripts/test_ssh_spaceport.py diff --git a/scripts/check_housekeeping.py b/scripts/check_housekeeping.py new file mode 100644 index 0000000..a20643c --- /dev/null +++ b/scripts/check_housekeeping.py @@ -0,0 +1,14 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" + +client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15) +stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods --field-selector=status.phase=Succeeded") +print("Completed pods:", stdout.read().decode('utf-8')) +client.close() + diff --git a/scripts/check_spaceport_net.py b/scripts/check_spaceport_net.py new file mode 100644 index 0000000..edcec38 --- /dev/null +++ b/scripts/check_spaceport_net.py @@ -0,0 +1,26 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +# Let's connect to Aerodrome master 140.44.4.11 and check reachability and DNS/ARP for Spaceport nodes +client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10) + +cmds = [ + "ping -c 2 140.44.4.41", + "ping -c 2 140.44.4.51", + "cat /etc/hosts | grep sninfr0", + "arp -an | grep 140.44.4.4", + "ip route" +] + +for cmd in cmds: + print(f"\n>>> Running on 140.44.4.11: {cmd}") + stdin, stdout, stderr = client.exec_command(cmd) + print(stdout.read().decode('utf-8').strip()) + +client.close() diff --git a/scripts/cleanup_spaceport.py b/scripts/cleanup_spaceport.py new file mode 100644 index 0000000..0a641f3 --- /dev/null +++ b/scripts/cleanup_spaceport.py @@ -0,0 +1,35 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +master_ip = "140.44.4.41" +print(f"Connecting to Spaceport Master: {master_ip}...") +client.connect(master_ip, username="root", key_filename=key_path, timeout=15) + +# A. Clean up completed pods across namespaces on Spaceport +print("\n>>> Pruning completed pods on Spaceport...") +stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl delete pod -A --field-selector=status.phase=Succeeded") +print(stdout.read().decode('utf-8')) +print(stderr.read().decode('utf-8')) + +# B. Save final etcd snapshot on Spaceport +snap_space = f"final-post-upgrade-spaceport-{int(time.time())}" +print(f"\n>>> Saving final etcd snapshot: {snap_space}...") +stdin, stdout, stderr = client.exec_command(f"k3s etcd-snapshot save --name {snap_space}") +print(stdout.read().decode('utf-8')) +print(stderr.read().decode('utf-8')) + +# C. List snapshots +print("\n>>> Recent etcd snapshots on Spaceport:") +stdin, stdout, stderr = client.exec_command("k3s etcd-snapshot list | tail -n 5") +print(stdout.read().decode('utf-8')) + +client.close() +print("\nSpaceport cleanup and snapshot completed successfully!") + diff --git a/scripts/post_upgrade_cleanup.py b/scripts/post_upgrade_cleanup.py new file mode 100644 index 0000000..9e141e9 --- /dev/null +++ b/scripts/post_upgrade_cleanup.py @@ -0,0 +1,75 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +def run_remote(ip, cmd): + print(f"\n[{ip}] >>> Running: {cmd}") + client.connect(ip, username="root", key_filename=key_path, timeout=15) + stdin, stdout, stderr = client.exec_command(cmd) + out = stdout.read().decode('utf-8') + err = stderr.read().decode('utf-8') + client.close() + if out: + print(out.strip()) + if err: + print(f"STDERR: {err.strip()}") + return out, err + +print("==================================================") +print("1. Cleaning up Aerodrome (140.44.4.11)...") +print("==================================================") + +# A. Clean up completed/failed helm-operation pods in cattle-system +cleanup_pods_cmd = ( + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system delete pod " + "--field-selector=status.phase=Succeeded" +) +run_remote("140.44.4.11", cleanup_pods_cmd) + +cleanup_failed_pods_cmd = ( + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system delete pod " + "--field-selector=status.phase=Failed" +) +run_remote("140.44.4.11", cleanup_failed_pods_cmd) + +# B. Clean up temporary files on Aerodrome +cleanup_files_cmd = "rm -rf /tmp/rancher-hop /tmp/rancher-test /tmp/rancher-2* /tmp/rancher_versions.txt" +run_remote("140.44.4.11", cleanup_files_cmd) + +# C. Save final etcd snapshot on Aerodrome +snap_aero = f"final-post-upgrade-aerodrome-{int(time.time())}" +run_remote("140.44.4.11", f"k3s etcd-snapshot save --name {snap_aero}") + +print("\n==================================================") +print("2. Cleaning up Spaceport (140.44.4.51)...") +print("==================================================") + +# A. Clean up completed pods across namespaces on Spaceport +cleanup_spaceport_pods = ( + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl delete pod -A " + "--field-selector=status.phase=Succeeded" +) +run_remote("140.44.4.51", cleanup_spaceport_pods) + +# B. Save final etcd snapshot on Spaceport +snap_space = f"final-post-upgrade-spaceport-{int(time.time())}" +run_remote("140.44.4.51", f"k3s etcd-snapshot save --name {snap_space}") + +print("\n==================================================") +print("3. Verification...") +print("==================================================") + +run_remote("140.44.4.11", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods") +run_remote("140.44.4.11", "k3s etcd-snapshot list | tail -n 5") +run_remote("140.44.4.51", "k3s etcd-snapshot list | tail -n 5") + +print("\n==================================================") +print("Post-upgrade cleanup completed successfully!") +print("==================================================") + diff --git a/scripts/test_direct_41.py b/scripts/test_direct_41.py new file mode 100644 index 0000000..39d19e3 --- /dev/null +++ b/scripts/test_direct_41.py @@ -0,0 +1,18 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +try: + print("Connecting directly to 140.44.4.41...") + client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5) + print("Connected successfully to 140.44.4.41!") + stdin, stdout, stderr = client.exec_command("hostname; k3s kubectl get nodes -o wide") + print(stdout.read().decode('utf-8')) + client.close() +except Exception as e: + print("Direct connect to 140.44.4.41 error:", e) diff --git a/scripts/test_spaceport_snap.py b/scripts/test_spaceport_snap.py new file mode 100644 index 0000000..1970c0e --- /dev/null +++ b/scripts/test_spaceport_snap.py @@ -0,0 +1,19 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("ssh -o StrictHostKeyChecking=no root@140.44.4.41 'k3s etcd-snapshot save --name final-post-upgrade-spaceport-$(date +%s)'") +print(stdout.read().decode('utf-8')) +print(stderr.read().decode('utf-8')) + +stdin, stdout, stderr = client.exec_command("ssh -o StrictHostKeyChecking=no root@140.44.4.41 'k3s etcd-snapshot list | tail -n 5'") +print(stdout.read().decode('utf-8')) +print(stderr.read().decode('utf-8')) + +client.close() diff --git a/scripts/test_ssh_spaceport.py b/scripts/test_ssh_spaceport.py new file mode 100644 index 0000000..06809df --- /dev/null +++ b/scripts/test_ssh_spaceport.py @@ -0,0 +1,18 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10) + +cmd = "ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.41 'hostname; k3s kubectl get nodes'" +print(f"\n>>> Running: {cmd}") +stdin, stdout, stderr = client.exec_command(cmd) +print("STDOUT:", stdout.read().decode('utf-8')) +print("STDERR:", stderr.read().decode('utf-8')) + +client.close()