81 lines
3.3 KiB
Python
81 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Phase 3 - Step 1:
|
|
1. Remove `is-default-class` from `local-path` on aerodrome and spaceport, leaving `longhorn` as the single default.
|
|
2. Force clear stuck `Terminating` PVCs on aerodrome and spaceport:
|
|
- aerodrome: infra/pihole, infra/technitium-longhorn-pvc
|
|
- spaceport: infrastructure/technitium-longhorn-pvc
|
|
"""
|
|
import os
|
|
import sys
|
|
import paramiko
|
|
|
|
sys.stdout.reconfigure(encoding="utf-8")
|
|
|
|
PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet")
|
|
pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH)
|
|
|
|
TARGETS = [
|
|
{
|
|
"name": "aerodrome",
|
|
"ip": "140.44.4.11",
|
|
"stuck_pvcs": [("infra", "pihole"), ("infra", "technitium-longhorn-pvc")]
|
|
},
|
|
{
|
|
"name": "spaceport",
|
|
"ip": "140.44.4.41",
|
|
"stuck_pvcs": [("infrastructure", "technitium-longhorn-pvc")]
|
|
}
|
|
]
|
|
|
|
for t in TARGETS:
|
|
print(f"\n==================================================")
|
|
print(f"REMEDIATING STORAGE ON {t['name']} ({t['ip']})")
|
|
print(f"==================================================")
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(t["ip"], username="root", pkey=pkey, timeout=10)
|
|
|
|
# 1. Unset local-path default annotation
|
|
print("1. Unsetting is-default-class from local-path...")
|
|
cmd_sc = """
|
|
kubectl annotate sc local-path storageclass.kubernetes.io/is-default-class- --overwrite 2>&1 || true
|
|
kubectl get sc
|
|
"""
|
|
stdin, stdout, stderr = client.exec_command(cmd_sc)
|
|
print(stdout.read().decode('utf-8', errors='replace'))
|
|
|
|
# 2. Clear stuck PVCs
|
|
print("2. Clearing stuck Terminating PVCs...")
|
|
for ns, pvc in t["stuck_pvcs"]:
|
|
print(f" Checking {ns}/{pvc}...")
|
|
# Check if volume name exists
|
|
cmd_vol = f"kubectl -n {ns} get pvc {pvc} -o jsonpath='{{.spec.volumeName}}' 2>/dev/null || true"
|
|
stdin, stdout, stderr = client.exec_command(cmd_vol)
|
|
vol_name = stdout.read().decode().strip()
|
|
|
|
# Patch finalizers to null
|
|
cmd_patch = f"""
|
|
kubectl -n {ns} patch pvc {pvc} -p '{{"metadata":{{"finalizers":null}}}}' 2>&1 || true
|
|
if [ -n "{vol_name}" ]; then
|
|
echo " Deleting associated Longhorn volume {vol_name} if present..."
|
|
kubectl -n longhorn-system patch lhs {vol_name} -p '{{"metadata":{{"finalizers":null}}}}' 2>/dev/null || true
|
|
kubectl -n longhorn-system patch lhv {vol_name} -p '{{"metadata":{{"finalizers":null}}}}' 2>/dev/null || true
|
|
kubectl -n longhorn-system delete lhv {vol_name} 2>/dev/null || true
|
|
fi
|
|
"""
|
|
stdin, stdout, stderr = client.exec_command(cmd_patch)
|
|
print(" ", stdout.read().decode('utf-8', errors='replace').strip())
|
|
|
|
# 3. Verify PVC status
|
|
print("3. Verifying remaining Terminating PVCs...")
|
|
cmd_verify = "kubectl get pvc -A | grep -i terminating || echo ' All clear! Zero stuck PVCs.'"
|
|
stdin, stdout, stderr = client.exec_command(cmd_verify)
|
|
print(stdout.read().decode('utf-8', errors='replace'))
|
|
|
|
client.close()
|
|
except Exception as e:
|
|
print(f"Error on {t['name']}: {e}")
|
|
|