feat: add cluster auditing, management, and upgrade scripts

scripts/diagnose_issues.py

@@ -0,0 +1,52 @@
+import paramiko
+
+def check_spaceport():
+    print("================ SPACEPORT (Coder & Semaphore) ================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    def run(cmd):
+        print(f"\n--- {cmd} ---")
+        _, out, err = ssh.exec_command(cmd)
+        res = out.read().decode().strip()
+        if res:
+            print(res)
+        e = err.read().decode().strip()
+        if e:
+            print("ERR:", e)
+            
+    run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide")
+    run("kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=40")
+    run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o wide")
+    run("kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=40")
+    run("curl -k -I https://coder.snarfnet.net")
+    run("curl -k -I https://semaphore.snarfnet.net")
+    ssh.close()
+
+def check_swarm():
+    print("\n================ SWARM (Peanut & NPM) ================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    def run(cmd):
+        print(f"\n--- {cmd} ---")
+        _, out, err = ssh.exec_command(cmd)
+        res = out.read().decode().strip()
+        if res:
+            print(res)
+        e = err.read().decode().strip()
+        if e:
+            print("ERR:", e)
+
+    run("sudo docker service ps peanut_peanut --no-trunc")
+    run("sudo docker service logs peanut_peanut --tail=40")
+    run("curl -s -I http://140.44.4.70:8050 || curl -s -I http://140.44.4.71:8050")
+    run("curl -k -I https://peanut.snarfnet.net || true")
+    run("sudo sqlite3 /mnt/nas_docker/npm/data/database.sqlite 'select id,domain_names,forward_host,forward_port,enabled from proxy_host where domain_names like \"%peanut%\" or domain_names like \"%coder%\" or domain_names like \"%semaphore%\";'")
+    ssh.close()
+
+check_spaceport()
+check_swarm()
+