feat: add cluster auditing, management, and upgrade scripts

scripts/persist_coder_storage.py

@@ -0,0 +1,131 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+    e = err.read().decode('utf-8', errors='replace').strip()
+    if e:
+        print("ERR:", e)
+
+print("[1] Creating backup directory on Spaceport master...")
+run("mkdir -p /var/backups/coder_home")
+
+print("[2] Getting current Coder pod name...")
+_, out, _ = ssh.exec_command("kubectl get pod -n infrastructure -l app.kubernetes.io/name=coder -o jsonpath='{.items[0].metadata.name}'")
+pod_name = out.read().decode().strip()
+print(f"Active Coder pod: {pod_name}")
+
+print("[3] Copying /home/coder from active pod to /var/backups/coder_home...")
+run(f"kubectl exec -n infrastructure {pod_name} -- tar -czf /tmp/coder_backup.tar.gz -C /home/coder .")
+run(f"kubectl cp infrastructure/{pod_name}:/tmp/coder_backup.tar.gz /var/backups/coder_home/coder_backup.tar.gz")
+run("ls -lh /var/backups/coder_home/coder_backup.tar.gz")
+
+print("[4] Creating PersistentVolumeClaim 'coder-data' (10Gi, Longhorn)...")
+pvc_yaml = """cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: coder-data
+  namespace: infrastructure
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
+EOF"""
+run(pvc_yaml)
+
+print("[5] Launching temporary population pod to unpack backup into PVC...")
+helper_yaml = """cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: coder-pvc-populator
+  namespace: infrastructure
+spec:
+  restartPolicy: Never
+  containers:
+  - name: populator
+    image: alpine:latest
+    command: ["sleep", "300"]
+    volumeMounts:
+    - name: data
+      mountPath: /data
+  volumes:
+  - name: data
+    persistentVolumeClaim:
+      claimName: coder-data
+EOF"""
+run(helper_yaml)
+
+print("Waiting for populator pod to be Ready...")
+run("kubectl wait --for=condition=Ready pod/coder-pvc-populator -n infrastructure --timeout=60s")
+
+print("[6] Copying tarball into populator pod and unpacking with correct permissions (1000:1000)...")
+run("kubectl cp /var/backups/coder_home/coder_backup.tar.gz infrastructure/coder-pvc-populator:/data/coder_backup.tar.gz")
+run("kubectl exec -n infrastructure coder-pvc-populator -- tar -xzf /data/coder_backup.tar.gz -C /data/")
+run("kubectl exec -n infrastructure coder-pvc-populator -- rm /data/coder_backup.tar.gz")
+run("kubectl exec -n infrastructure coder-pvc-populator -- chown -R 1000:1000 /data")
+run("kubectl exec -n infrastructure coder-pvc-populator -- ls -la /data")
+
+print("[7] Deleting helper populator pod...")
+run("kubectl delete pod coder-pvc-populator -n infrastructure --grace-period=0 --force")
+
+print("[8] Patching Coder deployment with persistent volume and strategy=Recreate...")
+patch_json = '''kubectl patch deployment coder -n infrastructure --patch '
+{
+  "spec": {
+    "strategy": {
+      "type": "Recreate",
+      "rollingUpdate": null
+    },
+    "template": {
+      "spec": {
+        "containers": [
+          {
+            "name": "coder",
+            "volumeMounts": [
+              {
+                "name": "coder-data",
+                "mountPath": "/home/coder"
+              }
+            ]
+          }
+        ],
+        "volumes": [
+          {
+            "name": "coder-data",
+            "persistentVolumeClaim": {
+              "claimName": "coder-data"
+            }
+          }
+        ]
+      }
+    }
+  }
+}' '''
+run(patch_json)
+
+print("[9] Awaiting Coder rollout...")
+run("kubectl rollout status deployment/coder -n infrastructure --timeout=180s")
+run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide")
+
+print("[10] Verifying Coder health and database persistence...")
+run("curl -s https://coder.snarfnet.net/api/v2/users/first")
+run("curl -k -s -I https://coder.snarfnet.net | head -n 8")
+
+ssh.close()
+