feat: add housekeeping and cleanup scripts for cluster maintenance

scripts/check_housekeeping.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods --field-selector=status.phase=Succeeded")
+print("Completed pods:", stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_spaceport_net.py

@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+# Let's connect to Aerodrome master 140.44.4.11 and check reachability and DNS/ARP for Spaceport nodes
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+cmds = [
+    "ping -c 2 140.44.4.41",
+    "ping -c 2 140.44.4.51",
+    "cat /etc/hosts | grep sninfr0",
+    "arp -an | grep 140.44.4.4",
+    "ip route"
+]
+
+for cmd in cmds:
+    print(f"\n>>> Running on 140.44.4.11: {cmd}")
+    stdin, stdout, stderr = client.exec_command(cmd)
+    print(stdout.read().decode('utf-8').strip())
+
+client.close()

scripts/cleanup_spaceport.py

@@ -0,0 +1,35 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+master_ip = "140.44.4.41"
+print(f"Connecting to Spaceport Master: {master_ip}...")
+client.connect(master_ip, username="root", key_filename=key_path, timeout=15)
+
+# A. Clean up completed pods across namespaces on Spaceport
+print("\n>>> Pruning completed pods on Spaceport...")
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl delete pod -A --field-selector=status.phase=Succeeded")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+# B. Save final etcd snapshot on Spaceport
+snap_space = f"final-post-upgrade-spaceport-{int(time.time())}"
+print(f"\n>>> Saving final etcd snapshot: {snap_space}...")
+stdin, stdout, stderr = client.exec_command(f"k3s etcd-snapshot save --name {snap_space}")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+# C. List snapshots
+print("\n>>> Recent etcd snapshots on Spaceport:")
+stdin, stdout, stderr = client.exec_command("k3s etcd-snapshot list | tail -n 5")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+print("\nSpaceport cleanup and snapshot completed successfully!")
+

scripts/post_upgrade_cleanup.py

@@ -0,0 +1,75 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+def run_remote(ip, cmd):
+    print(f"\n[{ip}] >>> Running: {cmd}")
+    client.connect(ip, username="root", key_filename=key_path, timeout=15)
+    stdin, stdout, stderr = client.exec_command(cmd)
+    out = stdout.read().decode('utf-8')
+    err = stderr.read().decode('utf-8')
+    client.close()
+    if out:
+        print(out.strip())
+    if err:
+        print(f"STDERR: {err.strip()}")
+    return out, err
+
+print("==================================================")
+print("1. Cleaning up Aerodrome (140.44.4.11)...")
+print("==================================================")
+
+# A. Clean up completed/failed helm-operation pods in cattle-system
+cleanup_pods_cmd = (
+    "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system delete pod "
+    "--field-selector=status.phase=Succeeded"
+)
+run_remote("140.44.4.11", cleanup_pods_cmd)
+
+cleanup_failed_pods_cmd = (
+    "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system delete pod "
+    "--field-selector=status.phase=Failed"
+)
+run_remote("140.44.4.11", cleanup_failed_pods_cmd)
+
+# B. Clean up temporary files on Aerodrome
+cleanup_files_cmd = "rm -rf /tmp/rancher-hop /tmp/rancher-test /tmp/rancher-2* /tmp/rancher_versions.txt"
+run_remote("140.44.4.11", cleanup_files_cmd)
+
+# C. Save final etcd snapshot on Aerodrome
+snap_aero = f"final-post-upgrade-aerodrome-{int(time.time())}"
+run_remote("140.44.4.11", f"k3s etcd-snapshot save --name {snap_aero}")
+
+print("\n==================================================")
+print("2. Cleaning up Spaceport (140.44.4.51)...")
+print("==================================================")
+
+# A. Clean up completed pods across namespaces on Spaceport
+cleanup_spaceport_pods = (
+    "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl delete pod -A "
+    "--field-selector=status.phase=Succeeded"
+)
+run_remote("140.44.4.51", cleanup_spaceport_pods)
+
+# B. Save final etcd snapshot on Spaceport
+snap_space = f"final-post-upgrade-spaceport-{int(time.time())}"
+run_remote("140.44.4.51", f"k3s etcd-snapshot save --name {snap_space}")
+
+print("\n==================================================")
+print("3. Verification...")
+print("==================================================")
+
+run_remote("140.44.4.11", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods")
+run_remote("140.44.4.11", "k3s etcd-snapshot list | tail -n 5")
+run_remote("140.44.4.51", "k3s etcd-snapshot list | tail -n 5")
+
+print("\n==================================================")
+print("Post-upgrade cleanup completed successfully!")
+print("==================================================")
+

scripts/test_direct_41.py

@@ -0,0 +1,18 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+try:
+    print("Connecting directly to 140.44.4.41...")
+    client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+    print("Connected successfully to 140.44.4.41!")
+    stdin, stdout, stderr = client.exec_command("hostname; k3s kubectl get nodes -o wide")
+    print(stdout.read().decode('utf-8'))
+    client.close()
+except Exception as e:
+    print("Direct connect to 140.44.4.41 error:", e)

scripts/test_spaceport_snap.py

@@ -0,0 +1,19 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("ssh -o StrictHostKeyChecking=no root@140.44.4.41 'k3s etcd-snapshot save --name final-post-upgrade-spaceport-$(date +%s)'")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("ssh -o StrictHostKeyChecking=no root@140.44.4.41 'k3s etcd-snapshot list | tail -n 5'")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+client.close()

scripts/test_ssh_spaceport.py

@@ -0,0 +1,18 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+cmd = "ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.41 'hostname; k3s kubectl get nodes'"
+print(f"\n>>> Running: {cmd}")
+stdin, stdout, stderr = client.exec_command(cmd)
+print("STDOUT:", stdout.read().decode('utf-8'))
+print("STDERR:", stderr.read().decode('utf-8'))
+
+client.close()