feat: add script to check cattle-system pods and events on spaceport server

scripts/audit_cluster_workloads.py

@@ -0,0 +1,21 @@
+import paramiko
+
+def inspect_cluster(ip, name):
+    print(f"\n=======================================================")
+    print(f"=== {name} WORKLOADS ({ip}) ===")
+    print(f"=======================================================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    cmd = "kubectl get deploy,sts,ds -A -o custom-columns=NAMESPACE:.metadata.namespace,KIND:.kind,NAME:.metadata.name,IMAGES:.spec.template.spec.containers[*].image"
+    _, out, _ = ssh.exec_command(cmd)
+    lines = out.read().decode().splitlines()
+    for l in lines:
+        if not any(k in l for k in ['kube-system', 'cattle-', 'metallb-system', 'longhorn-system']):
+            print(l)
+    ssh.close()
+
+inspect_cluster('140.44.4.11', 'AERODROME')
+inspect_cluster('140.44.4.41', 'SPACEPORT')
+

scripts/audit_gitops_manifests.py

@@ -0,0 +1,40 @@
+import os
+import re
+import paramiko
+
+manifest_root = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests"
+
+print("=== GITOPS MANIFEST IMAGE AUDIT ===")
+
+image_re = re.compile(r'image:\s*([^\s"\']+)', re.IGNORECASE)
+chart_re = re.compile(r'chart:\s*([^\s"\']+)', re.IGNORECASE)
+version_re = re.compile(r'version:\s*([^\s"\']+)', re.IGNORECASE)
+
+manifest_data = {}
+
+for root, dirs, files in os.walk(manifest_root):
+    for f in files:
+        if f.endswith('.yaml') or f.endswith('.yml'):
+            path = os.path.join(root, f)
+            rel_path = os.path.relpath(path, manifest_root)
+            with open(path, 'r', encoding='utf-8', errors='ignore') as fp:
+                content = fp.read()
+                images = image_re.findall(content)
+                charts = chart_re.findall(content)
+                versions = version_re.findall(content)
+                if images or charts:
+                    manifest_data[rel_path] = {
+                        'images': list(set(images)),
+                        'charts': list(set(charts)),
+                        'versions': list(set(versions))
+                    }
+
+for rel_path, data in sorted(manifest_data.items()):
+    print(f"\nFile: {rel_path}")
+    if data['images']:
+        print("  Images:", ", ".join(data['images']))
+    if data['charts']:
+        print("  Charts:", ", ".join(data['charts']))
+    if data['versions']:
+        print("  Versions:", ", ".join(data['versions']))
+

scripts/check_214_pods.py

@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -o wide")
+print("=== PODS IN CATTLE-SYSTEM ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
+print("=== ROLLOUT STATUS ===")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
+print("=== ENDPOINT ===")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+

scripts/check_215_pods.py

@@ -0,0 +1,29 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -o wide")
+print("=== PODS IN CATTLE-SYSTEM ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
+print("=== ROLLOUT STATUS ===")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
+print("=== ENDPOINT ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -n cattle-system")
+print("=== HELM LIST ===")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+

scripts/check_51.py

@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("ss -tulpn | grep ':80 '")
+run("ip a")
+run("kubectl get pod -A -o wide | grep sninfr4lnxwrk01 | grep pihole || true")
+ssh.close()
+

scripts/check_51_nat.py

@@ -0,0 +1,18 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("iptables -t nat -S | grep -E '80\b|31905|31937'")
+run("crictl ps")
+ssh.close()
+

scripts/check_aerodrome_pods.py

@@ -0,0 +1,15 @@
+import paramiko
+
+s = paramiko.SSHClient()
+s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, _ = s.exec_command(cmd)
+    print(out.read().decode())
+
+run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
+run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
+s.close()
+

scripts/check_aerodrome_preflight.py

@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("kubectl get nodes")
+run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
+run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness | grep -v healthy || true")
+ssh.close()
+

scripts/check_aerodrome_quick.py

@@ -0,0 +1,8 @@
+import paramiko
+ssh=paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+_,o,_=ssh.exec_command('kubectl get nodes')
+print(o.read().decode())
+ssh.close()
+

scripts/check_all_coder_snapshots.py

@@ -0,0 +1,24 @@
+import paramiko
+
+nodes = [
+    ("wrk01", "140.44.4.51"),
+    ("wrk02", "140.44.4.52"),
+    ("wrk03", "140.44.4.53"),
+    ("wrk04", "140.44.4.54"),
+    ("wrk05", "140.44.4.55"),
+    ("wrk06", "140.44.4.56"),
+]
+
+for name, ip in nodes:
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=4)
+        _, out, _ = ssh.exec_command("find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path '*/home/coder/.config/coderv2' 2>/dev/null")
+        res = out.read().decode().strip()
+        if res:
+            print(f"[{name} - {ip}] Found:\n{res}")
+        ssh.close()
+    except Exception as e:
+        print(f"[{name}] err: {e}")
+

scripts/check_all_endpoints.py

@@ -0,0 +1,19 @@
+import subprocess
+
+endpoints = [
+    ("Argo CD", "https://argo.snarfnet.net"),
+    ("Rancher Manager", "https://aerodrome.snarfnet.net"),
+    ("AWX Console", "https://awx.snarfnet.net"),
+    ("Authentik SSO", "https://authentik.snarfnet.net"),
+    ("Uptime Kuma", "https://uptime.snarfnet.net/dashboard"),
+    ("Gitea GitOps", "http://140.44.4.51:90"),
+    ("Zabbix Server", "http://140.44.4.51:8888"),
+    ("Portainer Server", "https://140.44.4.71:9443")
+]
+
+print("=== ENDPOINT HEALTH CHECK ===")
+for name, url in endpoints:
+    res = subprocess.run(f"curl.exe -k -s -o nul -w '%{{http_code}}' {url}", capture_output=True, text=True, shell=True)
+    code = res.stdout.strip()
+    print(f"[{code}] {name} ({url})")
+

scripts/check_apis.py

@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+
+test_cmd = """
+cd /tmp/rancher-test
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm template rancher ./rancher --namespace cattle-system --set hostname=aerodrome.snarfnet.net --set ingress.tls.source=secret | grep "apiVersion:" | sort | uniq -c
+"""
+
+stdin, stdout, stderr = client.exec_command(test_cmd)
+out = stdout.read().decode('utf-8')
+err = stderr.read().decode('utf-8')
+print("APIs used:")
+print(out)
+if err:
+    print("STDERR:", err)
+
+client.close()

scripts/check_argo_apps.py

@@ -0,0 +1,14 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("kubectl get applications -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
+print(stdout.read().decode())
+err = stderr.read().decode()
+if err:
+    print("ERR:", err)
+
+ssh.close()
+

scripts/check_argo_ing.py

@@ -0,0 +1,22 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("kubectl get ingress argo-ing -n infrastructure -o yaml")
+run("kubectl get svc pihole-web -n infrastructure -o yaml")
+run("kubectl get svc -n infrastructure")
+run("kubectl get svc -A | grep -E 'LoadBalancer|140.44.4.62'")
+run("curl -k -I -H 'Host: argo.snarfnet.net' http://140.44.4.62")
+run("curl -k -I -H 'Host: argo.snarfnet.net' https://140.44.4.62")
+ssh.close()
+

scripts/check_argocd_now.py

@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command("kubectl get applications -n infrastructure -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
+print(out.read().decode())
+
+ssh.close()
+

scripts/check_cattle_pods.py

@@ -0,0 +1,14 @@
+import paramiko
+
+def check_cattle(ip, cluster):
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    print(f"\n[{cluster}] Checking cattle-system pods...")
+    _, out, _ = ssh.exec_command("kubectl get pods -n cattle-system -o wide")
+    print(out.read().decode().strip())
+    ssh.close()
+
+check_cattle('140.44.4.11', 'AERODROME')
+check_cattle('140.44.4.41', 'SPACEPORT')
+

scripts/check_cattle_spaceport.py

@@ -0,0 +1,28 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+
+cmds = [
+    "k3s kubectl -n cattle-system get pods -o wide",
+    "k3s kubectl -n cattle-system describe pod -l app=cattle-cluster-agent",
+    "k3s kubectl get events -A --sort-by='.lastTimestamp' | tail -n 20"
+]
+
+for cmd in cmds:
+    print(f"\n==========================================")
+    print(f">>> Running on Spaceport: {cmd}")
+    print(f"==========================================")
+    stdin, stdout, stderr = client.exec_command(cmd)
+    print(stdout.read().decode('utf-8'))
+    err = stderr.read().decode('utf-8')
+    if err:
+        print("STDERR:", err)
+
+client.close()

scripts/check_chart_kubeversions.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+versions = [
+    "2.7.10", "2.8.5", "2.9.3", "2.10.3", "2.11.3", "2.12.3", "2.13.3", "2.14.3", "2.15.3"
+]
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+
+for ver in versions:
+    stdin, stdout, stderr = client.exec_command(f"KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm show chart rancher-stable/rancher --version {ver} | grep kubeVersion")
+    out = stdout.read().decode('utf-8').strip()
+    print(f"Rancher {ver}: {out}")
+
+client.close()
+

scripts/check_chart_versions.py

@@ -0,0 +1,13 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm search repo rancher-stable/rancher --versions > /tmp/rancher_versions.txt && head -n 30 /tmp/rancher_versions.txt")
+print(stdout.read().decode('utf-8'))
+client.close()

scripts/check_cluster_details.py

@@ -0,0 +1,13 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command('KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io -o wide')
+print(stdout.read().decode('utf-8'))
+client.close()

scripts/check_coder_logs.py

@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command('kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=30')
+print(out.read().decode())
+
+ssh.close()
+

scripts/check_coder_page.py

@@ -0,0 +1,14 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command("curl -s https://coder.snarfnet.net/api/v2/users/first")
+print("First user status:", out.read().decode())
+
+_, out2, _ = ssh.exec_command("curl -s https://coder.snarfnet.net/api/v2/experiments")
+print("Experiments:", out2.read().decode())
+
+ssh.close()
+

scripts/check_ghcr_tags.py

@@ -0,0 +1,24 @@
+import urllib.request
+import json
+
+def get_ghcr_tags(package):
+    # Public unauthenticated ghcr tags endpoint
+    url = f"https://ghcr.io/v2/{package}/tags/list"
+    try:
+        # Get anonymous token
+        token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{package}:pull"
+        with urllib.request.urlopen(token_url, timeout=5) as t_resp:
+            token = json.loads(t_resp.read().decode())['token']
+        
+        req = urllib.request.Request(url, headers={'Authorization': f'Bearer {token}'})
+        with urllib.request.urlopen(req, timeout=5) as resp:
+            data = json.loads(resp.read().decode())
+            tags = [t for t in data.get('tags', []) if not t.endswith('-dev') and not 'beta' in t]
+            return tags[-10:]
+    except Exception as e:
+        return [f"Err: {e}"]
+
+print("Homepage tags:", get_ghcr_tags("gethomepage/homepage"))
+print("Coder tags:", get_ghcr_tags("coder/coder"))
+print("Authentik tags:", get_ghcr_tags("goauthentik/server"))
+

scripts/check_grafana_spec.py

@@ -0,0 +1,15 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, _ = ssh.exec_command(cmd)
+    print(out.read().decode())
+
+run("kubectl get deployment grafana -n default -o jsonpath='{.spec.template.spec.containers[*].name}'")
+run("kubectl get deployment grafana -n default -o jsonpath='{.spec.template.spec.containers[*].image}'")
+ssh.close()
+

scripts/check_helm_cattle.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -n cattle-system")
+print(stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_helm_releases.py

@@ -0,0 +1,14 @@
+import paramiko
+
+def check_helm(ip, name):
+    print(f"\n=== {name} HELM RELEASES ({ip}) ===")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    _, out, _ = ssh.exec_command("helm list -A")
+    print(out.read().decode())
+    ssh.close()
+
+check_helm('140.44.4.11', 'AERODROME')
+check_helm('140.44.4.41', 'SPACEPORT')
+

scripts/check_historical_data.py

@@ -0,0 +1,60 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+# 1. Inspect Gotify database on Swarm
+print("=================== GOTIFY (Swarm 140.44.4.71) ===================")
+ssh_swm = paramiko.SSHClient()
+ssh_swm.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_swm.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run_swm(cmd):
+    _, out, err = ssh_swm.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+run_swm("cat c:\\Users\\Snarf\\OneDrive\\Desktop\\code\\SnarfCode\\docker-deployments\\gotify.yml 2>/dev/null || true")
+run_swm("sudo docker service inspect gotify_gotify --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")
+run_swm("sudo docker run --rm -v gotify_gotify_data:/data alpine ls -la /data 2>/dev/null || find /var/lib/docker/volumes/ -name 'gotify.db' 2>/dev/null")
+
+# Query gotify.db if found
+run_swm("""python3 -c '
+import sqlite3, glob
+db_files = glob.glob("/var/lib/docker/volumes/**/gotify.db", recursive=True)
+print("Gotify DB files:", db_files)
+for db in db_files:
+    try:
+        conn = sqlite3.connect(db)
+        cur = conn.cursor()
+        print("Tables in", db)
+        for row in cur.execute("SELECT name FROM sqlite_master WHERE type=\"table\""):
+            print(" ", row[0])
+        print("Applications count:", cur.execute("SELECT count(*) FROM applications").fetchone()[0])
+        print("Clients count:", cur.execute("SELECT count(*) FROM clients").fetchone()[0])
+        print("Messages count:", cur.execute("SELECT count(*) FROM messages").fetchone()[0])
+    except Exception as e:
+        print("Err reading", db, e)
+'""")
+
+ssh_swm.close()
+
+# 2. Inspect Semaphore BoltDB on Spaceport
+print("\n=================== SEMAPHORE (Spaceport 140.44.4.41) ===================")
+ssh_sp = paramiko.SSHClient()
+ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run_sp(cmd):
+    _, out, err = ssh_sp.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+run_sp("kubectl exec -n infrastructure deployment/semaphore -- ls -lh /var/lib/semaphore/database.boltdb")
+# Let's inspect projects and templates using semaphore CLI or strings
+run_sp("kubectl exec -n infrastructure deployment/semaphore -- strings /var/lib/semaphore/database.boltdb | grep -E 'project|playbook|inventory|repository' | head -n 30")
+
+ssh_sp.close()
+

scripts/check_homepage_spec.py

@@ -0,0 +1,15 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, _ = ssh.exec_command(cmd)
+    print(out.read().decode())
+
+run("kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath='{.spec.template.spec.containers[*].name}'")
+run("kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath='{.spec.template.spec.containers[*].image}'")
+ssh.close()
+

scripts/check_housekeeping.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods --field-selector=status.phase=Succeeded")
+print("Completed pods:", stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_jenkins.py

@@ -0,0 +1,23 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode().strip()
+    if res:
+        print(res)
+    e = err.read().decode().strip()
+    if e:
+        print("ERR:", e)
+
+run("kubectl get pod -n infrastructure jenkins-0 -o wide")
+run("kubectl describe pod -n infrastructure jenkins-0 | tail -n 25")
+run("curl -k -s -I https://jenkins.snarfnet.net | head -n 5")
+run("curl -s -I http://140.44.4.52:101 | head -n 5")
+
+ssh.close()
+

scripts/check_job134.py

@@ -0,0 +1,28 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=134)
+print(f"Job [134]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+events = job.job_events.filter(event__in=['runner_on_ok', 'runner_on_failed', 'playbook_on_stats']).order_by('-id')[:10]
+for e in events:
+    print(f"[{e.event}] Host: {e.event_data.get('host')} - Task: {e.event_data.get('task')}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+_, out, _ = ssh.exec_command('kubectl get nodes')
+print("=== KUBERNETES NODES ===")
+print(out.read().decode())
+
+ssh.close()
+

scripts/check_job134_end.py

@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=134)
+print(job.result_stdout[-1500:])
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job134_explain.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=134)
+print(f"Status: {job.status}")
+print(f"Job explanation: {job.job_explanation}")
+print(f"Result Traceback: {job.result_traceback}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job134_fail.py

@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=134)
+failed_events = job.job_events.filter(failed=True)
+for e in failed_events:
+    res = e.event_data.get('res', {})
+    msg = res.get('msg') or res.get('stderr') or res.get('stdout') or str(res)
+    print(f"Host: {e.event_data.get('host')}, Task: {e.event_data.get('task')}")
+    print(f"Details: {msg[:300]}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job134_stdout.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=134)
+lines = [l for l in job.result_stdout.split('\\n') if l.strip()]
+print('\\n'.join(lines[-35:]))
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job_status.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=131)
+print(f"Job [131]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+ssh.close()
+

scripts/check_k3s_channels.py

@@ -0,0 +1,12 @@
+import urllib.request
+import json
+
+url = "https://update.k3s.io/v1-release/channels"
+try:
+    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json'})
+    with urllib.request.urlopen(req, timeout=10) as resp:
+        data = json.loads(resp.read().decode())
+        for item in data.get('data', []):
+            print(f"{item.get('id')}: {item.get('latest')}")
+except Exception as e:
+    print("Error:", e)

scripts/check_logs_utf8.py

@@ -0,0 +1,32 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh_sp = paramiko.SSHClient()
+ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+ssh_swm = paramiko.SSHClient()
+ssh_swm.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_swm.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(ssh, cmd):
+    print(f"\n--- {cmd} ---")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+print("=== PEANUT LOGS (Swarm) ===")
+run(ssh_swm, "sudo docker service logs peanut_peanut --tail=50")
+
+print("\n=== CODER LOGS (Spaceport) ===")
+run(ssh_sp, "kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=50")
+
+print("\n=== SEMAPHORE LOGS (Spaceport) ===")
+run(ssh_sp, "kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=50")
+
+ssh_sp.close()
+ssh_swm.close()
+

scripts/check_longhorn_version.py

@@ -0,0 +1,15 @@
+import paramiko
+
+def check_lh(ip, cluster):
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    print(f"\n[{cluster}] Checking Longhorn images...")
+    _, out, _ = ssh.exec_command("kubectl get daemonset -n longhorn-system longhorn-manager -o jsonpath='{.spec.template.spec.containers[0].image}{\"\\n\"}'")
+    print("Manager image:", out.read().decode().strip())
+    _, out, _ = ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state | head -n 10")
+    print(out.read().decode().strip())
+    ssh.close()
+
+check_lh('140.44.4.11', 'AERODROME')
+check_lh('140.44.4.41', 'SPACEPORT')

scripts/check_npm_all_hosts.py

@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -E "server_name|set \\$server|set \\$port" /data/nginx/proxy_host/*.conf""")
+print(stdout.read().decode())
+
+ssh.close()
+

scripts/check_npm_argo.py

@@ -0,0 +1,21 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("sudo docker ps | grep nginx")
+run("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') find /data -maxdepth 3")
+run("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') cat /etc/hosts")
+run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -rn "argo" /etc/nginx/ /data/nginx/""")
+run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -rn "pihole" /etc/nginx/ /data/nginx/""")
+
+ssh.close()

scripts/check_npm_db.py

@@ -0,0 +1,15 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = "sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c \"python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print('\\n'.join(str(r) for r in cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE is_deleted=0')))\\\"\""
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+err = stderr.read().decode()
+if err:
+    print("ERR:", err)
+
+ssh.close()
+

scripts/check_npm_mounts.py

@@ -0,0 +1,15 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+
+run("sudo docker service inspect nginx-proxy-manager_app --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")
+run("sudo docker inspect $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{json .Mounts}}'")
+ssh.close()
+

scripts/check_npm_nodes.py

@@ -0,0 +1,14 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+
+run("sudo docker service ps nginx-proxy-manager_app --filter 'desired-state=running'")
+ssh.close()
+

scripts/check_npm_user.py

@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+# Check user table in NPM db
+run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print(list(cur.execute('SELECT id, name, email FROM user')))\\\"\"""")
+
+ssh.close()
+

scripts/check_old_coder_layer.py

@@ -0,0 +1,26 @@
+import paramiko
+
+nodes = [
+    ("ctl01", "140.44.4.41"),
+    ("wrk01", "140.44.4.51"),
+    ("wrk02", "140.44.4.52"),
+    ("wrk03", "140.44.4.53"),
+    ("wrk04", "140.44.4.54"),
+    ("wrk05", "140.44.4.55"),
+    ("wrk06", "140.44.4.56"),
+]
+
+for name, ip in nodes:
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
+        _, out, _ = ssh.exec_command("find /run/k3s/containerd/ /var/lib/rancher/k3s/agent/containerd/ -type d -name 'coderv2' 2>/dev/null")
+        res = out.read().decode().strip()
+        if res:
+            print(f"[{name} - {ip}] FOUND coderv2:\n{res}")
+        else:
+            print(f"[{name} - {ip}] none")
+        ssh.close()
+    except Exception as e:
+        print(f"[{name} - {ip}] err: {e}")

scripts/check_other_apps.py

@@ -0,0 +1,22 @@
+import paramiko
+
+def query_cluster(ip, name):
+    print(f"\n=================== CLUSTER {name} ({ip}) ===================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    def run(cmd):
+        print(f"=== {cmd} ===")
+        _, out, err = ssh.exec_command(cmd)
+        res = out.read().decode().strip()
+        if res:
+            print(res)
+    
+    run("kubectl get ns")
+    run("kubectl get deploy,sts -A | grep -E 'awx|authentik|zabbix|coder'")
+    ssh.close()
+
+query_cluster('140.44.4.11', 'AERODROME')
+query_cluster('140.44.4.41', 'SPACEPORT')
+

scripts/check_peanut_now.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+run("sudo docker service logs peanut_peanut --tail=25")
+run("curl -s -I http://140.44.4.70:8050/")
+run("curl -s http://140.44.4.70:8050/ | grep -i setup || echo 'NO SETUP REDIRECT'")
+
+ssh.close()
+

scripts/check_peanut_storage.py

@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n--- {cmd} ---")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+run("ls -la /mnt/swarm_shared/legacy_volumes/peanut/")
+run("ssh snshpd4lnxswm05 'ls -la /mnt/swarm_shared/legacy_volumes/peanut/'")
+run("sudo docker service inspect peanut_peanut")
+
+ssh.close()
+

scripts/check_pod_events.py

@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system describe pod rancher-574bb974f4-877rf | grep -A 10 -B 2 Events:")
+print("=== POD EVENTS ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system describe pod rancher-574bb974f4-877rf | grep -A 15 'Last State:'")
+print("=== LAST STATE ===")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+

scripts/check_pods_now.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -l app=rancher -o wide")
+print("=== RANCHER PODS ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_probe_values.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("grep -i -C 5 probe /tmp/rancher-2153/rancher/values.yaml")
+print("Probe in values.yaml:")
+print(stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_psp.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+
+test_cmd = """
+cd /tmp/rancher-test
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm template rancher ./rancher --namespace cattle-system --set hostname=aerodrome.snarfnet.net --set ingress.tls.source=secret | grep -B 2 -A 8 "policy/v1beta1"
+"""
+
+stdin, stdout, stderr = client.exec_command(test_cmd)
+out = stdout.read().decode('utf-8')
+print(out)
+client.close()
+

scripts/check_pvc_progress.py

@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command('kubectl get pods,pvc -n infrastructure | grep coder')
+print(out.read().decode())
+
+ssh.close()
+

scripts/check_rancher_clusters.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io")
+print("=== RANCHER MANAGEMENT CLUSTERS ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_rancher_compat.py

@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("kubectl get settings.management.cattle.io server-version -o yaml")
+run("kubectl get pod -n cattle-system -l app=rancher -o jsonpath='{.items[0].spec.containers[*].image}'")
+run("kubectl get clusters.management.cattle.io -o custom-columns=NAME:.metadata.name,DISPLAY:.spec.displayName,VERSION:.status.version.gitVersion")
+ssh.close()
+

scripts/check_rancher_final.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+
+print("=== CHECKING ALL CLUSTERS IN RANCHER ===")
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io")
+print(stdout.read().decode('utf-8'))
+
+print("=== CHECKING CATTLE-SYSTEM STATUS ===")
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+

scripts/check_rancher_helm_versions.py

@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+run("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo update rancher-stable")
+run("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm search repo rancher-stable/rancher --versions | head -n 35")
+
+ssh.close()

scripts/check_rancher_live.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -l app=rancher -o wide")
+print("=== RANCHER PODS ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
+print("=== ROLLOUT STATUS ===")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+client.close()
+

scripts/check_rancher_logs.py

@@ -0,0 +1,17 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system logs -l app=rancher -c rancher --tail=50")
+print("=== RANCHER LOGS ===")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+client.close()
+

scripts/check_rancher_versions.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm search repo rancher-stable/rancher --versions | head -n 35")
+print(stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_removal_script.py

@@ -0,0 +1,17 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+stdin, stdout, stderr = ssh.exec_command('cat /proc/3712358/cmdline')
+print("CMDLINE:", stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command('cat /usr/sbin/linux-check-removal || cat /usr/bin/linux-check-removal')
+print("SCRIPT:\n", stdout.read().decode())
+
+ssh.close()
+

scripts/check_revision_11.py

@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -l app=rancher -o wide")
+print("=== RANCHER PODS ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -l app=rancher-webhook -o wide")
+print("=== RANCHER WEBHOOK ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
+print("=== ENDPOINT ===")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+

scripts/check_semaphore_crash.py

@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode().strip()
+    errors = err.read().decode().strip()
+    if res:
+        print(res)
+    if errors:
+        print("ERR:", errors)
+
+run("kubectl logs deployment/semaphore -n infrastructure --tail=40")
+ssh.close()
+

scripts/check_semaphore_help.py

@@ -0,0 +1,14 @@
+import paramiko
+import time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+ssh.exec_command('kubectl run semaphore-test --image=semaphoreui/semaphore:v2.19.17 --restart=Never -n infrastructure --command -- strings /usr/bin/semaphore')
+time.sleep(6)
+_, out, _ = ssh.exec_command('kubectl logs semaphore-test -n infrastructure | grep -E "mysql|postgres|bolt" | head -n 30')
+print(out.read().decode())
+ssh.exec_command('kubectl delete pod semaphore-test -n infrastructure --grace-period=0 --force')
+ssh.close()
+

scripts/check_semaphore_stable.py

@@ -0,0 +1,9 @@
+import urllib.request
+import json
+
+url = "https://hub.docker.com/v2/repositories/semaphoreui/semaphore/tags?page_size=100"
+req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+with urllib.request.urlopen(req, timeout=5) as resp:
+    data = json.loads(resp.read().decode())
+    ga_tags = [t['name'] for t in data.get('results', []) if not any(x in t['name'] for x in ['-rc', '-beta', '-alpha', '-debug', 'latest'])]
+    print("Semaphore GA tags:", ga_tags[:10])

scripts/check_spaceport_health.py

@@ -0,0 +1,16 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+
+run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
+run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
+run("kubectl get applications -A -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
+ssh.close()
+

scripts/check_spaceport_lb.py

@@ -0,0 +1,18 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("kubectl get ds -A")
+run("kubectl get svc traefik -n kube-system -o yaml")
+ssh.close()
+

scripts/check_spaceport_net.py

@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+# Let's connect to Aerodrome master 140.44.4.11 and check reachability and DNS/ARP for Spaceport nodes
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+cmds = [
+    "ping -c 2 140.44.4.41",
+    "ping -c 2 140.44.4.51",
+    "cat /etc/hosts | grep sninfr0",
+    "arp -an | grep 140.44.4.4",
+    "ip route"
+]
+
+for cmd in cmds:
+    print(f"\n>>> Running on 140.44.4.11: {cmd}")
+    stdin, stdout, stderr = client.exec_command(cmd)
+    print(stdout.read().decode('utf-8').strip())
+
+client.close()

scripts/check_spaceport_packages.py

@@ -0,0 +1,31 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+spaceport_nodes = [
+    ('sninfr0lnxctl01', '140.44.4.41'),
+    ('sninfr0lnxctl02', '140.44.4.42'),
+    ('sninfr0lnxctl03', '140.44.4.43'),
+    ('sninfr0lnxctl04', '140.44.4.44'),
+    ('sninfr0lnxwrk01', '140.44.4.51'),
+    ('sninfr0lnxwrk02', '140.44.4.52'),
+    ('sninfr0lnxwrk03', '140.44.4.53'),
+    ('sninfr0lnxwrk04', '140.44.4.54'),
+    ('sninfr0lnxwrk05', '140.44.4.55'),
+    ('sninfr0lnxwrk06', '140.44.4.56')
+]
+
+for name, ip in spaceport_nodes:
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=5)
+        stdin, stdout, stderr = ssh.exec_command('apt list --upgradable 2>/dev/null | grep -v "Listing" | wc -l')
+        cnt = stdout.read().decode().strip()
+        print(f"[{name}] ({ip}): {cnt} packages upgradeable")
+        ssh.close()
+    except Exception as e:
+        print(f"[{name}] ({ip}): Error connecting: {e}")
+

scripts/check_spaceport_quick.py

@@ -0,0 +1,9 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+_, out, _ = ssh.exec_command('kubectl get nodes')
+print(out.read().decode())
+ssh.close()
+

scripts/check_startup_probe.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get deployment rancher -o jsonpath='{.spec.template.spec.containers[0].startupProbe}'")
+print("Startup Probe:", stdout.read().decode('utf-8'))
+client.close()
+

scripts/check_tls_rancher.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get secret tls-rancher-ingress")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+client.close()
+

scripts/check_upstream_tags.py

@@ -0,0 +1,18 @@
+import urllib.request
+import json
+
+def get_dockerhub_latest(repo, prefix=""):
+    url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=15"
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        with urllib.request.urlopen(req, timeout=5) as resp:
+            data = json.loads(resp.read().decode())
+            tags = [t['name'] for t in data.get('results', []) if not t['name'].endswith('-debug') and not 'beta' in t['name']]
+            return tags[:8]
+    except Exception as e:
+        return [f"Err: {e}"]
+
+print("Grafana tags:", get_dockerhub_latest("grafana/grafana"))
+print("Uptime Kuma tags:", get_dockerhub_latest("louislam/uptime-kuma"))
+print("Semaphore tags:", get_dockerhub_latest("semaphoreui/semaphore"))
+

scripts/check_upstream_tags2.py

@@ -0,0 +1,16 @@
+import urllib.request
+import json
+
+def get_tags(repo):
+    url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=15"
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        with urllib.request.urlopen(req, timeout=5) as resp:
+            data = json.loads(resp.read().decode())
+            tags = [t['name'] for t in data.get('results', []) if not t['name'].endswith('-debug') and not 'beta' in t['name']]
+            return tags[:8]
+    except Exception as e:
+        return [f"Err: {e}"]
+
+print("Jenkins tags:", get_tags("jenkins/jenkins"))
+

scripts/check_wrk03_fail.py

@@ -0,0 +1,27 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_check = """
+from awx.main.models import Job
+job = Job.objects.get(id=134)
+events = job.job_events.filter(event='runner_on_failed')
+for e in events:
+    print(f"FAILED Host: {e.event_data.get('host')}, Task: {e.event_data.get('task')}")
+    print(f"Res: {e.event_data.get('res')}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+_, out, _ = ssh.exec_command('kubectl get pods -A --field-selector spec.nodeName=sninfr2lnxwrk03')
+print("Pods on wrk03:\n", out.read().decode())
+
+ssh.close()
+

scripts/check_zabbix_status.py

@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command('kubectl get pods -n monitoring -o wide')
+print(out.read().decode())
+
+ssh.close()
+

scripts/check_zabbix_tags.py

@@ -0,0 +1,19 @@
+import urllib.request
+import json
+
+tags_to_check = [
+    "zabbix-server-pgsql",
+    "zabbix-agent2",
+    "zabbix-web-nginx-pgsql",
+    "zabbix-web-service"
+]
+
+for repo in tags_to_check:
+    url = f"https://registry.hub.docker.com/v2/repositories/zabbix/{repo}/tags/ubuntu-7.0.32"
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        with urllib.request.urlopen(req) as resp:
+            print(f"[OK] zabbix/{repo}:ubuntu-7.0.32 exists")
+    except Exception as e:
+        print(f"[ERR] zabbix/{repo}:ubuntu-7.0.32 -> {e}")
+

scripts/cleanup_spaceport.py

@@ -0,0 +1,35 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+master_ip = "140.44.4.41"
+print(f"Connecting to Spaceport Master: {master_ip}...")
+client.connect(master_ip, username="root", key_filename=key_path, timeout=15)
+
+# A. Clean up completed pods across namespaces on Spaceport
+print("\n>>> Pruning completed pods on Spaceport...")
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl delete pod -A --field-selector=status.phase=Succeeded")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+# B. Save final etcd snapshot on Spaceport
+snap_space = f"final-post-upgrade-spaceport-{int(time.time())}"
+print(f"\n>>> Saving final etcd snapshot: {snap_space}...")
+stdin, stdout, stderr = client.exec_command(f"k3s etcd-snapshot save --name {snap_space}")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+# C. List snapshots
+print("\n>>> Recent etcd snapshots on Spaceport:")
+stdin, stdout, stderr = client.exec_command("k3s etcd-snapshot list | tail -n 5")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+print("\nSpaceport cleanup and snapshot completed successfully!")
+

scripts/cleanup_unknown_pods.py

@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+for pod in ['pihole-85b7bd696d-ltrvg', 'technitium-dns-6dcf8fc9f6-mxd7p']:
+    cmd = f'kubectl patch pod -n infra {pod} -p "{{\\"metadata\\":{{\\"finalizers\\":null}}}}" --type=merge'
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"Patch {pod}:", stdout.read().decode().strip(), stderr.read().decode().strip())
+    
+    del_cmd = f'kubectl delete pod -n infra {pod} --force --grace-period=0'
+    stdin, stdout, stderr = ssh.exec_command(del_cmd)
+    print(f"Delete {pod}:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+# Check remaining pods in infra
+_, out, _ = ssh.exec_command('kubectl get pods -n infra')
+print("\nInfra pods:\n", out.read().decode())
+
+ssh.close()
+

scripts/comprehensive_health_sweep.py

@@ -0,0 +1,45 @@
+import urllib.request
+import ssl
+import sys
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+endpoints = [
+    ("Rancher Console", "https://aerodrome.snarfnet.net"),
+    ("Argo CD Web UI", "https://argo.snarfnet.net"),
+    ("Portainer Server", "https://140.44.4.71:9443"),
+    ("Authentik SSO Portal", "https://authentik.snarfnet.net"),
+    ("AWX Controller", "https://awx.snarfnet.net"),
+    ("Grafana Dashboards", "https://grafana.snarfnet.net"),
+    ("Homepage Dashboard", "https://homepage.snarfnet.net"),
+    ("Uptime Kuma Monitor", "https://uptime.snarfnet.net/dashboard"),
+    ("Jenkins Controller", "https://jenkins.snarfnet.net/login"),
+    ("Coder Workspace", "https://coder.snarfnet.net"),
+    ("Gitea GitOps Service", "http://140.44.4.51:90"),
+    ("Zabbix Web Console", "http://140.44.4.51:8888"),
+    ("Heimdall Portal", "http://140.44.4.71:85"),
+    ("Peanut Dashboard", "http://140.44.4.71:8050"),
+    ("Homebox Inventory", "http://140.44.4.71:3100"),
+    ("Semaphore UI", "https://semaphore.snarfnet.net"),
+    ("NPM Proxy Admin", "http://140.44.4.71:81")
+]
+
+print(f"{'SERVICE':<25} {'URL':<40} {'STATUS':<15}")
+print("=" * 80)
+
+for name, url in endpoints:
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        with urllib.request.urlopen(req, timeout=8, context=ctx) as resp:
+            code = resp.getcode()
+            print(f"{name:<25} {url:<40} {code} OK")
+    except urllib.error.HTTPError as e:
+        # Some endpoints return 401 or 403 on root if auth required
+        if e.code in [200, 301, 302, 307, 401, 403]:
+            print(f"{name:<25} {url:<40} {e.code} Responding")
+        else:
+            print(f"{name:<25} {url:<40} {e.code} HTTP Error")
+    except Exception as e:
+        print(f"{name:<25} {url:<40} ERR: {e}")

scripts/deep_search_coder.py

@@ -0,0 +1,20 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.52', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+print("Searching for Coder postgres data on wrk02 (140.44.4.52)...")
+cmd = """find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -name "coderv2" 2>/dev/null"""
+_, out, err = ssh.exec_command(cmd)
+res = out.read().decode().strip()
+if res:
+    print("Found coderv2 paths:\n", res)
+else:
+    print("No coderv2 found directly. Searching for pg_control...")
+    cmd2 = """find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path "*coderv2*" 2>/dev/null | head -n 20"""
+    _, out2, _ = ssh.exec_command(cmd2)
+    print("Search result:\n", out2.read().decode().strip())
+
+ssh.close()
+

scripts/diagnose_argo.py

@@ -0,0 +1,68 @@
+import socket
+import paramiko
+
+def check_dns():
+    print("--- DNS Check ---")
+    try:
+        ip = socket.gethostbyname("argo.snarfnet.net")
+        print(f"argo.snarfnet.net resolves to: {ip}")
+    except Exception as e:
+        print(f"DNS lookup error: {e}")
+
+def check_npm():
+    print("\n--- NPM Database Check (140.44.4.71) ---")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    # Check sqlite db or active nginx config in npm
+    cmd = """sudo python3 -c "import sqlite3; con = sqlite3.connect('/mnt/nfs_shares/nginx_proxy_manager/database.sqlite'); cur = con.cursor(); print(list(cur.execute(\\"SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled, is_deleted FROM proxy_host WHERE domain_names LIKE '%argo%'\\")))" """
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    out = stdout.read().decode().strip()
+    err = stderr.read().decode().strip()
+    print("NPM DB Proxy Host for argo:")
+    print(out)
+    if err:
+        print("Error/stderr:", err)
+        
+    # Check generated nginx config files
+    cmd2 = "sudo grep -rn 'argo.snarfnet.net' /mnt/nfs_shares/nginx_proxy_manager/nginx/proxy_host/ || true"
+    stdin, stdout, stderr = ssh.exec_command(cmd2)
+    print("NPM config match:")
+    print(stdout.read().decode().strip())
+    
+    ssh.close()
+
+def check_spaceport_argo():
+    print("\n--- Spaceport ArgoCD K8s Check (140.44.4.41) ---")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    cmd = "kubectl get ingress,svc,pod -n infrastructure | grep -i argo"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print("Spaceport Argo Resources:")
+    print(stdout.read().decode().strip())
+    
+    # Check Ingress YAML
+    cmd2 = "kubectl get ingress -n infrastructure -o yaml"
+    stdin, stdout, stderr = ssh.exec_command(cmd2)
+    ingress_yaml = stdout.read().decode().strip()
+    print("\nAll Ingresses in infrastructure:")
+    for line in ingress_yaml.splitlines():
+        if "name:" in line or "host:" in line or "service:" in line or "port:" in line:
+            print("  ", line)
+            
+    ssh.close()
+
+if __name__ == '__main__':
+    check_dns()
+    try:
+        check_npm()
+    except Exception as e:
+        print("NPM check failed:", e)
+    try:
+        check_spaceport_argo()
+    except Exception as e:
+        print("Spaceport check failed:", e)
+

scripts/diagnose_issues.py

@@ -0,0 +1,52 @@
+import paramiko
+
+def check_spaceport():
+    print("================ SPACEPORT (Coder & Semaphore) ================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    def run(cmd):
+        print(f"\n--- {cmd} ---")
+        _, out, err = ssh.exec_command(cmd)
+        res = out.read().decode().strip()
+        if res:
+            print(res)
+        e = err.read().decode().strip()
+        if e:
+            print("ERR:", e)
+            
+    run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide")
+    run("kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=40")
+    run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o wide")
+    run("kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=40")
+    run("curl -k -I https://coder.snarfnet.net")
+    run("curl -k -I https://semaphore.snarfnet.net")
+    ssh.close()
+
+def check_swarm():
+    print("\n================ SWARM (Peanut & NPM) ================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    def run(cmd):
+        print(f"\n--- {cmd} ---")
+        _, out, err = ssh.exec_command(cmd)
+        res = out.read().decode().strip()
+        if res:
+            print(res)
+        e = err.read().decode().strip()
+        if e:
+            print("ERR:", e)
+
+    run("sudo docker service ps peanut_peanut --no-trunc")
+    run("sudo docker service logs peanut_peanut --tail=40")
+    run("curl -s -I http://140.44.4.70:8050 || curl -s -I http://140.44.4.71:8050")
+    run("curl -k -I https://peanut.snarfnet.net || true")
+    run("sudo sqlite3 /mnt/nas_docker/npm/data/database.sqlite 'select id,domain_names,forward_host,forward_port,enabled from proxy_host where domain_names like \"%peanut%\" or domain_names like \"%coder%\" or domain_names like \"%semaphore%\";'")
+    ssh.close()
+
+check_spaceport()
+check_swarm()
+

scripts/diagnose_wrk07.py

@@ -0,0 +1,23 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== NODES ===")
+_, out, _ = ssh.exec_command('kubectl get nodes')
+print(out.read().decode())
+
+print("=== PODS ON SNINFR2LNXWRK07 ===")
+_, out, _ = ssh.exec_command('kubectl get pods -A -o wide | grep sninfr2lnxwrk07')
+print(out.read().decode())
+
+print("=== ALL PODS NOT RUNNING / NOT COMPLETED ===")
+_, out, _ = ssh.exec_command('kubectl get pods -A | grep -v Running | grep -v Completed')
+print(out.read().decode())
+
+ssh.close()
+

scripts/export_gitops_manifests.py

@@ -0,0 +1,93 @@
+import paramiko
+import yaml
+import os
+
+CLEAN_METADATA_KEYS = {'uid', 'resourceVersion', 'generation', 'creationTimestamp', 'managedFields', 'selfLink'}
+CLEAN_ANNOTATIONS_PREFIXES = {
+    'kubectl.kubernetes.io/last-applied-configuration',
+    'deployment.kubernetes.io/revision',
+    'field.cattle.io/publicEndpoints'
+}
+
+def clean_doc(doc):
+    if not isinstance(doc, dict):
+        return doc
+    doc.pop('status', None)
+    meta = doc.get('metadata', {})
+    for k in CLEAN_METADATA_KEYS:
+        meta.pop(k, None)
+    ann = meta.get('annotations', {})
+    keys_to_del = [k for k in ann if any(k.startswith(p) for p in CLEAN_ANNOTATIONS_PREFIXES)]
+    for k in keys_to_del:
+        ann.pop(k, None)
+    if not ann:
+        meta.pop('annotations', None)
+    return doc
+
+def dump_resources(ssh, query_cmd, out_file):
+    print(f"Querying: {query_cmd}")
+    _, out, err = ssh.exec_command(query_cmd)
+    raw = out.read().decode('utf-8', errors='replace')
+    if not raw.strip():
+        print(f"  Warning: No resources found for {query_cmd}")
+        return
+    
+    docs = yaml.safe_load_all(raw)
+    cleaned = []
+    for d in docs:
+        if not d: continue
+        if d.get('kind') == 'List':
+            for item in d.get('items', []):
+                cleaned.append(clean_doc(item))
+        else:
+            cleaned.append(clean_doc(d))
+            
+    if cleaned:
+        os.makedirs(os.path.dirname(out_file), exist_ok=True)
+        with open(out_file, 'w', encoding='utf-8') as fp:
+            yaml.dump_all(cleaned, fp, default_flow_style=False, sort_keys=False)
+        print(f"  Saved {len(cleaned)} resources -> {out_file}")
+
+# 1. Connect to Spaceport
+print("\n--- Connecting to Spaceport (140.44.4.41) ---")
+ssh_sp = paramiko.SSHClient()
+ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+base_sp = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport"
+
+# Homepage
+dump_resources(ssh_sp, "kubectl get deploy,svc,ing,cm -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o yaml", os.path.join(base_sp, "homepage", "homepage.yaml"))
+# Jenkins
+dump_resources(ssh_sp, "kubectl get sts,svc,ing -n infrastructure -l app.kubernetes.io/name=jenkins -o yaml", os.path.join(base_sp, "jenkins", "jenkins.yaml"))
+# Semaphore
+dump_resources(ssh_sp, "kubectl get deploy,svc,ing,cm -n infrastructure -l app.kubernetes.io/name=semaphore -o yaml", os.path.join(base_sp, "semaphore", "semaphore.yaml"))
+# Coder
+dump_resources(ssh_sp, "kubectl get deploy,svc,ing -n infrastructure -l app.kubernetes.io/name=coder -o yaml", os.path.join(base_sp, "coder", "coder.yaml"))
+# Gitea
+dump_resources(ssh_sp, "kubectl get deploy,sts,svc,ing,cm -n infrastructure -l app.kubernetes.io/instance=gitea -o yaml", os.path.join(base_sp, "gitea", "gitea.yaml"))
+# Zabbix
+dump_resources(ssh_sp, "kubectl get deploy,sts,svc,ing -n monitoring -l app.kubernetes.io/instance=zabbix -o yaml", os.path.join(base_sp, "zabbix", "zabbix.yaml"))
+
+ssh_sp.close()
+
+# 2. Connect to Aerodrome
+print("\n--- Connecting to Aerodrome (140.44.4.11) ---")
+ssh_ae = paramiko.SSHClient()
+ssh_ae.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_ae.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+base_ae = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome"
+
+# Grafana
+dump_resources(ssh_ae, "kubectl get deploy,svc,ing -n default -l app.kubernetes.io/name=grafana -o yaml", os.path.join(base_ae, "grafana", "grafana.yaml"))
+# Uptime Kuma
+dump_resources(ssh_ae, "kubectl get deploy,svc,ing -n monitoring -l app.kubernetes.io/name=uptime-kuma -o yaml", os.path.join(base_ae, "uptime-kuma", "uptime-kuma.yaml"))
+# AWX
+dump_resources(ssh_ae, "kubectl get awx,deploy,svc,ing -n awx -o yaml", os.path.join(base_ae, "awx", "awx.yaml"))
+# Authentik
+dump_resources(ssh_ae, "kubectl get deploy,sts,svc,ing -n infra -l app.kubernetes.io/instance=authentik -o yaml", os.path.join(base_ae, "authentik", "authentik.yaml"))
+
+ssh_ae.close()
+
+print("\nManifest extraction complete!")

scripts/find_coder_data.py

@@ -0,0 +1,20 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.52', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode('utf-8', errors='replace').strip()
+    if res:
+        print(res)
+
+run("crictl ps -a")
+run("find /var/lib/rancher/k3s/agent/containerd/ -type d -name 'coderv2' 2>/dev/null")
+run("find /run/k3s/containerd/ -type d -name 'coderv2' 2>/dev/null")
+run("find / -name 'database.boltdb' 2>/dev/null")
+
+ssh.close()
+

scripts/find_rancher_series.py

@@ -0,0 +1,27 @@
+import urllib.request
+import json
+import re
+
+releases = {}
+
+for page in range(1, 20):
+    url = f"https://api.github.com/repos/rancher/rancher/tags?per_page=100&page={page}"
+    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+    try:
+        data = json.loads(urllib.request.urlopen(req).read().decode())
+    except:
+        break
+    if not data:
+        break
+    for item in data:
+        name = item['name']
+        m = re.match(r"^v(2\.\d+)\.(\d+)$", name)
+        if m:
+            series = m.group(1)
+            patch = int(m.group(2))
+            if series not in releases or patch > releases[series]:
+                releases[series] = patch
+
+for s in sorted(releases.keys(), key=lambda x: [int(p) for p in x.split('.')]):
+    print(f"Series v{s}: latest patch is v{s}.{releases[s]}")
+

scripts/fix_and_upgrade_uptime.py

@@ -0,0 +1,36 @@
+import paramiko
+import time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode().strip()
+    if res:
+        print(res)
+    e = err.read().decode().strip()
+    if e:
+        print("ERR:", e)
+
+# 1. Update strategy to Recreate
+run("""kubectl patch deployment uptime-kuma-2-1756668961 -n monitoring --type='json' -p='[{"op": "replace", "path": "/spec/strategy", "value": {"type": "Recreate"}}]'""")
+
+# 2. Scale down to 0 to release all locks cleanly
+print("[*] Scaling down to 0 to release all volume locks...")
+run("kubectl scale deployment uptime-kuma-2-1756668961 -n monitoring --replicas=0")
+run("kubectl wait --for=delete pod -l app.kubernetes.io/name=uptime-kuma -n monitoring --timeout=60s || true")
+
+# 3. Set image to 2.5.9
+run("kubectl set image deployment/uptime-kuma-2-1756668961 uptime-kuma=louislam/uptime-kuma:2.5.9 -n monitoring")
+
+# 4. Scale up to 1
+print("[*] Scaling up to 1...")
+run("kubectl scale deployment uptime-kuma-2-1756668961 -n monitoring --replicas=1")
+run("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=120s")
+run("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma -o wide")
+
+ssh.close()
+

scripts/fix_argo_npm.py

@@ -0,0 +1,58 @@
+import paramiko
+
+def fix_argo():
+    # 1. Update on swm01 (140.44.4.71)
+    ssh1 = paramiko.SSHClient()
+    ssh1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh1.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    print("--- Backing up SQLite DB and 39.conf ---")
+    cmd_backup = """
+    sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "
+      cp /data/database.sqlite /data/database.sqlite.bak.argo &&
+      cp /data/nginx/proxy_host/39.conf /data/nginx/proxy_host/39.conf.bak
+    "
+    """
+    stdin, stdout, stderr = ssh1.exec_command(cmd_backup)
+    print("Backup stdout:", stdout.read().decode())
+    print("Backup stderr:", stderr.read().decode())
+    
+    print("--- Updating SQLite DB ---")
+    cmd_db = """
+    sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "
+      python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); cur.execute('UPDATE proxy_host SET forward_host=\\\'140.44.4.62\\\' WHERE id=39'); con.commit(); con.close(); print('DB updated successfully')\\\"
+    "
+    """
+    stdin, stdout, stderr = ssh1.exec_command(cmd_db)
+    print("DB stdout:", stdout.read().decode())
+    print("DB stderr:", stderr.read().decode())
+    
+    print("--- Updating 39.conf ---")
+    cmd_conf = """
+    sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "
+      sed -i 's/140.44.4.51/140.44.4.62/g' /data/nginx/proxy_host/39.conf
+    "
+    """
+    stdin, stdout, stderr = ssh1.exec_command(cmd_conf)
+    print("Conf sed stdout:", stdout.read().decode())
+    print("Conf sed stderr:", stderr.read().decode())
+    
+    print("--- Reloading NGINX on swm01 ---")
+    stdin, stdout, stderr = ssh1.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
+    print("Reload swm01 stdout:", stdout.read().decode())
+    print("Reload swm01 stderr:", stderr.read().decode())
+    ssh1.close()
+
+    # 2. Reload on swm02 (140.44.4.72)
+    ssh2 = paramiko.SSHClient()
+    ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh2.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    print("--- Reloading NGINX on swm02 ---")
+    stdin, stdout, stderr = ssh2.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
+    print("Reload swm02 stdout:", stdout.read().decode())
+    print("Reload swm02 stderr:", stderr.read().decode())
+    ssh2.close()
+
+if __name__ == '__main__':
+    fix_argo()
+

scripts/fix_cluster_issues.py

@@ -0,0 +1,34 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+# 1. Install open-iscsi on sninfr2lnxwrk01 (140.44.4.21)
+print("=== INSTALLING OPEN-ISCSI ON WRK01 (140.44.4.21) ===")
+ssh_wrk01 = paramiko.SSHClient()
+ssh_wrk01.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_wrk01.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+stdin, stdout, stderr = ssh_wrk01.exec_command('apt-get update -qq && apt-get install -y open-iscsi nfs-common && systemctl enable --now iscsid')
+print(stdout.read().decode())
+print("iscsid status:", end=" ")
+_, out, _ = ssh_wrk01.exec_command('systemctl is-active iscsid')
+print(out.read().decode().strip())
+ssh_wrk01.close()
+
+# 2. Check on Master
+ssh_master = paramiko.SSHClient()
+ssh_master.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh_master.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== DELETING CRASHING LONGHORN-MANAGER-SQTL9 POD SO IT RECREATES WITH ISCSID ===")
+ssh_master.exec_command('kubectl delete pod -n longhorn-system longhorn-manager-sqtl9')
+
+print("=== CHECKING HELM-INSTALL-TRAEFIK-CRD LOGS ===")
+_, out, _ = ssh_master.exec_command('kubectl logs -n kube-system helm-install-traefik-crd-jt9zk --tail=25')
+print(out.read().decode())
+
+print("=== CLEANING 130-DAY UNKNOWN PODS IN INFRA ===")
+ssh_master.exec_command('kubectl delete pod -n infra pihole-85b7bd696d-ltrvg technitium-dns-6dcf8fc9f6-mxd7p --force --grace-period=0')
+
+ssh_master.close()
+

scripts/fix_traefik_and_verify.py

@@ -0,0 +1,41 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== FIXING TRAEFIK CRD LABELS & ANNOTATIONS ===")
+annotate_cmd = """
+for crd in $(kubectl get crd -o name | grep 'traefik.io'); do
+  echo "Patching $crd..."
+  kubectl annotate $crd meta.helm.sh/release-name=traefik-crd meta.helm.sh/release-namespace=kube-system --overwrite
+  kubectl label $crd app.kubernetes.io/managed-by=Helm --overwrite
+done
+"""
+stdin, stdout, stderr = ssh.exec_command(annotate_cmd)
+print(stdout.read().decode())
+
+print("=== DELETING FAILED HELM INSTALL JOBS TO RETRIGGER ===")
+ssh.exec_command('kubectl delete job -n kube-system helm-install-traefik-crd helm-install-traefik')
+
+print("Waiting 15 seconds for Helm controller to reconcile...")
+time.sleep(15)
+
+print("=== CHECKING LONGHORN-MANAGER ON WRK01 ===")
+_, out, _ = ssh.exec_command('kubectl get pods -n longhorn-system -l app=longhorn-manager -o wide')
+print(out.read().decode())
+
+print("=== CHECKING ALL PODS IN KUBE-SYSTEM ===")
+_, out, _ = ssh.exec_command('kubectl get pods -n kube-system | grep traefik')
+print(out.read().decode())
+
+print("=== CHECKING ANY REMAINING CRASHING PODS ===")
+_, out, _ = ssh.exec_command('kubectl get pods -A | grep -v Running | grep -v Completed')
+print(out.read().decode())
+
+ssh.close()
+

scripts/fix_uptime_kuma.py

@@ -0,0 +1,18 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    print(out.read().decode().strip())
+    e = err.read().decode().strip()
+    if e:
+        print("ERR:", e)
+
+run("kubectl get deployment uptime-kuma-2-1756668961 -n monitoring -o yaml | grep -A 4 strategy:")
+run("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma")
+ssh.close()
+

scripts/get_coder_env.py

@@ -0,0 +1,21 @@
+import paramiko
+import json
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, err = ssh.exec_command('kubectl get pod -n infrastructure -l app.kubernetes.io/name=coder -o json')
+pod = json.loads(out.read().decode())
+if pod.get('items'):
+    container = pod['items'][0]['spec']['containers'][0]
+    print("Image:", container.get('image'))
+    print("Env:")
+    for e in container.get('env', []):
+        print(f"  {e.get('name')}: {e.get('value', e.get('valueFrom'))}")
+    print("VolumeMounts:")
+    for vm in container.get('volumeMounts', []):
+        print(f"  {vm.get('name')} -> {vm.get('mountPath')}")
+
+ssh.close()
+

scripts/get_rancher_values.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values rancher -n cattle-system")
+print("=== CURRENT RANCHER VALUES ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+

scripts/inspect_argocd_apps.py

@@ -0,0 +1,15 @@
+import glob
+import yaml
+
+files = glob.glob(r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\argocd\**\*.yaml", recursive=True)
+for f in files:
+    with open(f, 'r') as fp:
+        docs = yaml.safe_load_all(fp)
+        for doc in docs:
+            if not doc: continue
+            if doc.get('kind') == 'Application':
+                name = doc.get('metadata', {}).get('name')
+                dest = doc.get('spec', {}).get('destination', {})
+                src = doc.get('spec', {}).get('source', {})
+                print(f"App: {name:<30} Path: {src.get('path'):<35} Server: {dest.get('server'):<35} NS: {dest.get('namespace')}")
+

scripts/inspect_authentik.py

@@ -0,0 +1,18 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode().strip()
+    if res:
+        print(res)
+
+run("kubectl get deploy,sts,pod -n infra -o wide")
+run("kubectl get deploy authentik-server -n infra -o yaml | grep -E 'image:|strategy:' -A 5")
+
+ssh.close()
+

scripts/inspect_coder_data_to_persist.py

@@ -0,0 +1,23 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, err = ssh.exec_command(cmd)
+    res = out.read().decode().strip()
+    if res:
+        print(res)
+    e = err.read().decode().strip()
+    if e:
+        print("ERR:", e)
+
+run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder")
+run("kubectl exec -n infrastructure deployment/coder -- ls -la /home/coder")
+run("kubectl exec -n infrastructure deployment/coder -- du -sh /home/coder/.config/coderv2")
+run("curl -s https://coder.snarfnet.net/api/v2/users/first")
+
+ssh.close()
+

scripts/inspect_coder_helm.py

@@ -0,0 +1,21 @@
+import paramiko
+import base64
+import gzip
+import json
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command('kubectl get secret -n infrastructure sh.helm.release.v1.coder.v5 -o jsonpath="{.data.release}"')
+raw = out.read().decode().strip()
+if raw:
+    data = base64.b64decode(raw)
+    data = base64.b64decode(data) # helm encodes twice
+    decomp = gzip.decompress(data)
+    rel = json.loads(decomp)
+    print("Chart:", rel.get('chart', {}).get('metadata', {}).get('version'))
+    print("Values:", json.dumps(rel.get('config', {}), indent=2))
+
+ssh.close()
+