feat: add infrastructure automation scripts and master improvement plan

2026-11-02 03:07:29 -05:00
parent d1a3ba988a
commit 9c3de8bb4d
94 changed files with 3232 additions and 49 deletions
--- a/scripts/fix_storageclasses_and_stuck_pvcs.py
+++ b/scripts/fix_storageclasses_and_stuck_pvcs.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""
+Phase 3 - Step 1:
+1. Remove `is-default-class` from `local-path` on aerodrome and spaceport, leaving `longhorn` as the single default.
+2. Force clear stuck `Terminating` PVCs on aerodrome and spaceport:
+   - aerodrome: infra/pihole, infra/technitium-longhorn-pvc
+   - spaceport: infrastructure/technitium-longhorn-pvc
+"""
+import os
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet")
+pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH)
+
+TARGETS = [
+    {
+        "name": "aerodrome",
+        "ip": "140.44.4.11",
+        "stuck_pvcs": [("infra", "pihole"), ("infra", "technitium-longhorn-pvc")]
+    },
+    {
+        "name": "spaceport",
+        "ip": "140.44.4.41",
+        "stuck_pvcs": [("infrastructure", "technitium-longhorn-pvc")]
+    }
+]
+
+for t in TARGETS:
+    print(f"\n==================================================")
+    print(f"REMEDIATING STORAGE ON {t['name']} ({t['ip']})")
+    print(f"==================================================")
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(t["ip"], username="root", pkey=pkey, timeout=10)
+        
+        # 1. Unset local-path default annotation
+        print("1. Unsetting is-default-class from local-path...")
+        cmd_sc = """
+        kubectl annotate sc local-path storageclass.kubernetes.io/is-default-class- --overwrite 2>&1 || true
+        kubectl get sc
+        """
+        stdin, stdout, stderr = client.exec_command(cmd_sc)
+        print(stdout.read().decode('utf-8', errors='replace'))
+        
+        # 2. Clear stuck PVCs
+        print("2. Clearing stuck Terminating PVCs...")
+        for ns, pvc in t["stuck_pvcs"]:
+            print(f"  Checking {ns}/{pvc}...")
+            # Check if volume name exists
+            cmd_vol = f"kubectl -n {ns} get pvc {pvc} -o jsonpath='{{.spec.volumeName}}' 2>/dev/null || true"
+            stdin, stdout, stderr = client.exec_command(cmd_vol)
+            vol_name = stdout.read().decode().strip()
+            
+            # Patch finalizers to null
+            cmd_patch = f"""
+            kubectl -n {ns} patch pvc {pvc} -p '{{"metadata":{{"finalizers":null}}}}' 2>&1 || true
+            if [ -n "{vol_name}" ]; then
+                echo "    Deleting associated Longhorn volume {vol_name} if present..."
+                kubectl -n longhorn-system patch lhs {vol_name} -p '{{"metadata":{{"finalizers":null}}}}' 2>/dev/null || true
+                kubectl -n longhorn-system patch lhv {vol_name} -p '{{"metadata":{{"finalizers":null}}}}' 2>/dev/null || true
+                kubectl -n longhorn-system delete lhv {vol_name} 2>/dev/null || true
+            fi
+            """
+            stdin, stdout, stderr = client.exec_command(cmd_patch)
+            print("   ", stdout.read().decode('utf-8', errors='replace').strip())
+
+        # 3. Verify PVC status
+        print("3. Verifying remaining Terminating PVCs...")
+        cmd_verify = "kubectl get pvc -A | grep -i terminating || echo '  All clear! Zero stuck PVCs.'"
+        stdin, stdout, stderr = client.exec_command(cmd_verify)
+        print(stdout.read().decode('utf-8', errors='replace'))
+        
+        client.close()
+    except Exception as e:
+        print(f"Error on {t['name']}: {e}")
+