#!/usr/bin/env python3 """ Phase 3 - Step 1: 1. Remove `is-default-class` from `local-path` on aerodrome and spaceport, leaving `longhorn` as the single default. 2. Force clear stuck `Terminating` PVCs on aerodrome and spaceport: - aerodrome: infra/pihole, infra/technitium-longhorn-pvc - spaceport: infrastructure/technitium-longhorn-pvc """ import os import sys import paramiko sys.stdout.reconfigure(encoding="utf-8") PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet") pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH) TARGETS = [ { "name": "aerodrome", "ip": "140.44.4.11", "stuck_pvcs": [("infra", "pihole"), ("infra", "technitium-longhorn-pvc")] }, { "name": "spaceport", "ip": "140.44.4.41", "stuck_pvcs": [("infrastructure", "technitium-longhorn-pvc")] } ] for t in TARGETS: print(f"\n==================================================") print(f"REMEDIATING STORAGE ON {t['name']} ({t['ip']})") print(f"==================================================") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(t["ip"], username="root", pkey=pkey, timeout=10) # 1. Unset local-path default annotation print("1. Unsetting is-default-class from local-path...") cmd_sc = """ kubectl annotate sc local-path storageclass.kubernetes.io/is-default-class- --overwrite 2>&1 || true kubectl get sc """ stdin, stdout, stderr = client.exec_command(cmd_sc) print(stdout.read().decode('utf-8', errors='replace')) # 2. Clear stuck PVCs print("2. Clearing stuck Terminating PVCs...") for ns, pvc in t["stuck_pvcs"]: print(f" Checking {ns}/{pvc}...") # Check if volume name exists cmd_vol = f"kubectl -n {ns} get pvc {pvc} -o jsonpath='{{.spec.volumeName}}' 2>/dev/null || true" stdin, stdout, stderr = client.exec_command(cmd_vol) vol_name = stdout.read().decode().strip() # Patch finalizers to null cmd_patch = f""" kubectl -n {ns} patch pvc {pvc} -p '{{"metadata":{{"finalizers":null}}}}' 2>&1 || true if [ -n "{vol_name}" ]; then echo " Deleting associated Longhorn volume {vol_name} if present..." kubectl -n longhorn-system patch lhs {vol_name} -p '{{"metadata":{{"finalizers":null}}}}' 2>/dev/null || true kubectl -n longhorn-system patch lhv {vol_name} -p '{{"metadata":{{"finalizers":null}}}}' 2>/dev/null || true kubectl -n longhorn-system delete lhv {vol_name} 2>/dev/null || true fi """ stdin, stdout, stderr = client.exec_command(cmd_patch) print(" ", stdout.read().decode('utf-8', errors='replace').strip()) # 3. Verify PVC status print("3. Verifying remaining Terminating PVCs...") cmd_verify = "kubectl get pvc -A | grep -i terminating || echo ' All clear! Zero stuck PVCs.'" stdin, stdout, stderr = client.exec_command(cmd_verify) print(stdout.read().decode('utf-8', errors='replace')) client.close() except Exception as e: print(f"Error on {t['name']}: {e}")