feat: add infrastructure audit and diagnostic helper scripts

2026-11-06 12:37:49 -05:00
parent 06c203485f
commit aea8608f3d
44 changed files with 1141 additions and 1 deletions
--- a/2
+++ b/2
--- a/scripts/check_43_access.py
+++ b/scripts/check_43_access.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-43_access.log")
+print("=== Access log 43 (pihole2) around 11:00 ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_55_and_43.py
+++ b/scripts/check_55_and_43.py
@@ -0,0 +1,17 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/55.conf")
+print("=== 55.conf (pihole1) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/43.conf")
+print("=== 43.conf (pihole2) ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_all_providers.py
+++ b/scripts/check_all_providers.py
@@ -0,0 +1,30 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url):
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        return json.loads(resp.read().decode('utf-8'))
+
+apps = api_call("https://authentik.snarfnet.net/api/v3/core/applications/?page_size=100")
+for a in apps.get('results', []):
+    print(f"App: {a['name']:30} | Slug: {a['slug']:15} | Provider: {a.get('provider')}")
+
+providers = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/?page_size=100")
+for p in providers.get('results', []):
+    print(f"Proxy Provider: {p['name']:30} | PK: {p['pk']:4} | Mode: {p['mode']:15} | Cookie Domain: {p.get('cookie_domain')}")
+
--- a/scripts/check_authentik_logs.py
+++ b/scripts/check_authentik_logs.py
@@ -0,0 +1,19 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=key_path, timeout=10)
+
+# Check Authentik server logs
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl logs -n infra deployment/authentik-server --tail=50")
+print("Authentik server logs:")
+print(stdout.read().decode())
+
+# Check Authentik outpost logs if any
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n infra")
+print("Infra pods on Aerodrome:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_npm_cert.py
+++ b/scripts/check_npm_cert.py
@@ -0,0 +1,17 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) openssl x509 -in /etc/letsencrypt/live/npm-1/fullchain.pem -noout -dates -subject -issuer")
+print("=== Certificate npm-1 ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) openssl x509 -in /etc/letsencrypt/live/npm-1/fullchain.pem -noout -ext subjectAltName")
+print("=== SAN npm-1 ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_npm_logs.py
+++ b/scripts/check_npm_logs.py
@@ -0,0 +1,31 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+# Check error logs for pihole1 (55) and pihole2 (43)
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-43_error.log 2>/dev/null || true")
+print("=== Error log 43 (pihole2) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-43_access.log 2>/dev/null || true")
+print("=== Access log 43 (pihole2) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-55_error.log 2>/dev/null || true")
+print("=== Error log 55 (pihole1) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-55_access.log 2>/dev/null || true")
+print("=== Access log 55 (pihole1) ===")
+print(stdout.read().decode())
+
+# Check default error log
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/fallback_error.log 2>/dev/null || true")
+print("=== Fallback error log ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_npm_pihole_confs.py
+++ b/scripts/check_npm_pihole_confs.py
@@ -0,0 +1,23 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+# Check all proxy configs in NPM
+stdin, stdout, stderr = ssh.exec_command("sudo grep -rn 'pihole' /data/nginx/proxy_host/ 2>/dev/null || sudo grep -rn 'pihole' /etc/nginx/conf.d/ 2>/dev/null || sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) grep -rn 'pihole' /data/nginx/proxy_host/")
+print("Grep pihole in NPM:")
+print(stdout.read().decode())
+
+# Check the exact config files for pihole1 and pihole2
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/52.conf 2>/dev/null || true")
+print("Config 52 (or similar):")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/53.conf 2>/dev/null || true")
+print("Config 53 (or similar):")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_sem_app.py
+++ b/scripts/check_sem_app.py
@@ -0,0 +1,25 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/core/applications/semaphore/", headers=headers)
+with urllib.request.urlopen(req, context=ctx) as resp:
+    app = json.loads(resp.read().decode('utf-8'))
+    print("Semaphore App in Authentik:")
+    print("  launch_url:", app.get('launch_url'))
+    print("  meta_launch_url:", app.get('meta_launch_url'))
+    print("  provider:", app.get('provider'))
+
--- a/scripts/check_sem_auth_strings.py
+++ b/scripts/check_sem_auth_strings.py
@@ -0,0 +1,17 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- strings /usr/local/bin/semaphore | grep -i '/api/auth/' | head -n 30")
+print("Auth routes in semaphore:")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- strings /usr/local/bin/semaphore | grep -i 'oidc' | head -n 30")
+print("OIDC strings in semaphore:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_sem_oidc_routes.py
+++ b/scripts/check_sem_oidc_routes.py
@@ -0,0 +1,18 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- strings /usr/local/bin/semaphore | grep -E '/api/auth/oidc|/auth/oidc'")
+print("OIDC routes in Semaphore binary:")
+print(stdout.read().decode())
+
+# Check how Semaphore starts OIDC
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl logs -n infrastructure deployment/semaphore --tail=50")
+print("Semaphore container logs:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_sem_pod_oidc.py
+++ b/scripts/check_sem_pod_oidc.py
@@ -0,0 +1,17 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- env | grep -i oidc || true")
+print("OIDC env in Semaphore pod:")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- cat /etc/semaphore/config.json 2>/dev/null || true")
+print("/etc/semaphore/config.json:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_sem_ps.py
+++ b/scripts/check_sem_ps.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- ps aux")
+print("ps aux in Semaphore:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_sem_routes.py
+++ b/scripts/check_sem_routes.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n infrastructure deployment/semaphore -- strings /usr/bin/semaphore 2>/dev/null || k3s kubectl exec -n infrastructure deployment/semaphore -- strings $(which semaphore) 2>/dev/null | grep -i 'oidc' | head -n 30")
+print("Semaphore oidc strings:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_semaphore_cm.py
+++ b/scripts/check_semaphore_cm.py
@@ -0,0 +1,18 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get cm -n infrastructure semaphore-config -o yaml")
+print("=== Semaphore ConfigMap ===")
+print(stdout.read().decode())
+
+# Also check semaphore Ingress
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get ingress -n infrastructure semaphore-ing -o yaml")
+print("=== Semaphore Ingress ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_svclb_spaceport.py
+++ b/scripts/check_svclb_spaceport.py
@@ -0,0 +1,17 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get ds -n kube-system")
+print("Daemonsets in kube-system on Spaceport:")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl describe pod -n kube-system svclb-pihole-web-vip-dafeaccd-2qvp9 | grep -E 'Node:|Events:|failed'")
+print("Describe pending svclb pod:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_swarm_sudo.py
+++ b/scripts/check_swarm_sudo.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'")
+print("Docker containers on 140.44.4.71:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_syno_npm_logs.py
+++ b/scripts/check_syno_npm_logs.py
@@ -0,0 +1,25 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 25 /data/logs/proxy-host-11_access.log")
+print("=== Access log 11 (sn-nas) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 25 /data/logs/proxy-host-11_error.log")
+print("=== Error log 11 (sn-nas) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 25 /data/logs/proxy-host-28_access.log")
+print("=== Access log 28 (snarfnas) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 25 /data/logs/proxy-host-28_error.log")
+print("=== Error log 28 (snarfnas) ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_synology_html.py
+++ b/scripts/check_synology_html.py
@@ -0,0 +1,17 @@
+import urllib.request
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+url = "https://sn-nas.snarfnet.net/"
+req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        content = resp.read().decode('utf-8', errors='ignore')
+        print("HTML length:", len(content))
+        print("HTML contents:\n", content)
+except Exception as e:
+    print("Error:", e)
+
--- a/scripts/check_uptime_npm.py
+++ b/scripts/check_uptime_npm.py
@@ -0,0 +1,20 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) grep -rn 'uptime.snarfnet.net' /data/nginx/proxy_host/")
+print("Grep uptime.snarfnet.net in NPM:")
+print(stdout.read().decode())
+
+# Print the file
+for line in stdout.read().decode().strip().splitlines():
+    f = line.split(":")[0]
+    stdin2, stdout2, stderr2 = ssh.exec_command(f"sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat {f}")
+    print(f"=== {f} ===")
+    print(stdout2.read().decode())
+
+ssh.close()
+
--- a/scripts/check_vault_rancher.py
+++ b/scripts/check_vault_rancher.py
@@ -0,0 +1,38 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+def check_vault_and_rancher():
+    print("=== Checking Rancher and Vault ===")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.11', username='root', key_filename=key_path, timeout=10)
+
+    # Check rancher pods and ingress
+    stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -A | grep -E 'rancher|vault' || true")
+    print("Rancher/Vault pods on Aerodrome:")
+    print(stdout.read().decode())
+
+    stdin, stdout, stderr = ssh.exec_command("k3s kubectl get ingress -A | grep -E 'rancher|vault' || true")
+    print("Rancher/Vault ingress on Aerodrome:")
+    print(stdout.read().decode())
+
+    ssh.close()
+
+    ssh2 = paramiko.SSHClient()
+    ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh2.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+    stdin, stdout, stderr = ssh2.exec_command("k3s kubectl get pods -A | grep -E 'rancher|vault' || true")
+    print("Rancher/Vault pods on Spaceport:")
+    print(stdout.read().decode())
+
+    # Check pending svclb pods on Spaceport
+    stdin, stdout, stderr = ssh2.exec_command("k3s kubectl get pods -n kube-system | grep svclb-pihole || true")
+    print("svclb pods on Spaceport:")
+    print(stdout.read().decode())
+
+    ssh2.close()
+
+check_vault_and_rancher()
+
--- a/scripts/check_wan_certs.py
+++ b/scripts/check_wan_certs.py
@@ -0,0 +1,23 @@
+import socket
+import ssl
+
+def check_cert(domain, ip="174.106.252.96"):
+    print(f"=== Checking SSL for {domain} at {ip}:443 ===")
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = True
+    try:
+        with socket.create_connection((ip, 443), timeout=5) as sock:
+            with ctx.wrap_socket(sock, server_hostname=domain) as ssock:
+                cert = ssock.getpeercert()
+                print("Cert Subject:", cert.get('subject'))
+                print("SAN:", cert.get('subjectAltName'))
+                print("Cipher:", ssock.cipher())
+                print("ALPN:", ssock.selected_alpn_protocol())
+    except Exception as e:
+        print("SSL Error:", e)
+
+check_cert('pihole1.snarfnet.net')
+check_cert('pihole2.snarfnet.net')
+check_cert('dns1.snarfnet.net')
+check_cert('dns2.snarfnet.net')
+
--- a/scripts/cleanup_spaceport_svclb.py
+++ b/scripts/cleanup_spaceport_svclb.py
@@ -0,0 +1,25 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+with open(r'homelab-gitops-repo/manifests/spaceport/dns/dns-stack.yaml', 'r') as f:
+    manifest_content = f.read()
+
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl apply -f -')
+stdin.write(manifest_content)
+stdin.channel.shutdown_write()
+print("Apply:", stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl delete svc -n dns pihole-web-vip 2>/dev/null || true')
+print("Delete old svc:", stdout.read().decode())
+
+# Check pending pods on Spaceport now
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers")
+print("Pending/failing pods on Spaceport:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/full_infra_audit.py
+++ b/scripts/full_infra_audit.py
@@ -0,0 +1,158 @@
+import paramiko
+import urllib.request
+import ssl
+import json
+import socket
+import concurrent.futures
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+def check_k8s_cluster(host, name):
+    res = {'name': name, 'host': host, 'errors': [], 'warnings': []}
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(host, username='root', key_filename=key_path, timeout=10)
+
+        # Check nodes
+        stdin, stdout, stderr = ssh.exec_command("k3s kubectl get nodes --no-headers")
+        nodes_out = stdout.read().decode().strip().splitlines()
+        not_ready = [n for n in nodes_out if not " Ready" in n]
+        res['total_nodes'] = len(nodes_out)
+        res['not_ready_nodes'] = not_ready
+
+        # Check failing pods across all namespaces
+        stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers")
+        failing_pods = stdout.read().decode().strip().splitlines()
+        res['failing_pods'] = failing_pods
+
+        # Check DNS namespace pods
+        stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -o wide --no-headers")
+        dns_pods = stdout.read().decode().strip().splitlines()
+        res['dns_pods'] = dns_pods
+
+        # Check DNS service VIP
+        stdin, stdout, stderr = ssh.exec_command("k3s kubectl get svc -n dns -o wide --no-headers")
+        dns_svc = stdout.read().decode().strip().splitlines()
+        res['dns_svc'] = dns_svc
+
+        ssh.close()
+    except Exception as e:
+        res['errors'].append(f"SSH failed: {e}")
+    return res
+
+def check_swarm():
+    res = {'name': 'Docker Swarm / NPM', 'host': '140.44.4.71', 'errors': []}
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+        stdin, stdout, stderr = ssh.exec_command("docker node ls --format '{{.Hostname}}: {{.Status}} ({{.Availability}})'")
+        nodes = stdout.read().decode().strip().splitlines()
+        res['nodes'] = nodes
+
+        stdin, stdout, stderr = ssh.exec_command("docker ps --format '{{.Names}}: {{.Status}}'")
+        containers = stdout.read().decode().strip().splitlines()
+        res['containers'] = containers
+
+        ssh.close()
+    except Exception as e:
+        res['errors'].append(f"SSH failed: {e}")
+    return res
+
+def test_dns_resolution(server_ip, domain="google.com"):
+    import struct
+    try:
+        # Construct raw DNS query packet
+        # Transaction ID: 0x1234, Flags: standard query (0x0100), Questions: 1
+        header = struct.pack(">HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0)
+        qname = b"".join(bytes([len(p)]) + p.encode('ascii') for p in domain.split(".")) + b"\x00"
+        question = qname + struct.pack(">HH", 1, 1) # Type A, Class IN
+        packet = header + question
+
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.settimeout(3.0)
+        sock.sendto(packet, (server_ip, 53))
+        data, _ = sock.recvfrom(512)
+        sock.close()
+        # Check RCODE in response
+        flags = struct.unpack(">H", data[2:4])[0]
+        rcode = flags & 0x000F
+        return {'status': 'OK', 'rcode': rcode, 'bytes': len(data)}
+    except Exception as e:
+        return {'status': 'FAIL', 'error': str(e)}
+
+def check_endpoint(url):
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+
+    req = urllib.request.Request(url, headers={'User-Agent': 'AntigravityAudit/2.0'})
+    try:
+        # Don't follow redirects to capture 302/200 properly
+        class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
+            def http_error_302(self, req, fp, code, msg, headers):
+                return fp
+            http_error_301 = http_error_302
+            http_error_303 = http_error_302
+            http_error_307 = http_error_302
+            http_error_308 = http_error_302
+
+        opener = urllib.request.build_opener(NoRedirectHandler)
+        with opener.open(req, timeout=5) as resp:
+            code = resp.status if hasattr(resp, 'status') else resp.code
+            location = resp.headers.get('Location', '')
+            server = resp.headers.get('Server', '')
+            return {'url': url, 'code': code, 'location': location, 'server': server, 'ok': code in [200, 301, 302, 307, 308]}
+    except urllib.error.HTTPError as e:
+        return {'url': url, 'code': e.code, 'error': str(e), 'ok': e.code in [200, 301, 302, 307, 308, 401, 403]}
+    except Exception as e:
+        return {'url': url, 'code': 0, 'error': str(e), 'ok': False}
+
+print("=== STARTING FULL INFRASTRUCTURE & ENDPOINT AUDIT ===")
+
+# 1. Check K8s Clusters
+aero = check_k8s_cluster('140.44.4.11', 'Aerodrome')
+space = check_k8s_cluster('140.44.4.41', 'Spaceport')
+swarm = check_swarm()
+
+# 2. Check DNS Port 53 on both VIPs
+dns_aero = test_dns_resolution('140.44.4.36', 'google.com')
+dns_space = test_dns_resolution('140.44.4.65', 'google.com')
+dns_local = test_dns_resolution('140.44.4.36', 'authentik.snarfnet.net')
+
+# 3. Check Web Endpoints
+endpoints_to_check = [
+    'https://authentik.snarfnet.net',
+    'https://pihole1.snarfnet.net/admin/',
+    'https://pihole2.snarfnet.net/admin/',
+    'https://dns1.snarfnet.net',
+    'https://dns2.snarfnet.net',
+    'https://git.snarfnet.net',
+    'https://vault.snarfnet.net',
+    'https://rancher.snarfnet.net',
+    'https://argocd.snarfnet.net',
+    'https://zabbix.snarfnet.net',
+    'https://semaphore.snarfnet.net',
+    'https://kuma.snarfnet.net',
+    'https://homepage.snarfnet.net',
+    'https://grafana.snarfnet.net'
+]
+
+endpoint_results = []
+with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+    endpoint_results = list(executor.map(check_endpoint, endpoints_to_check))
+
+audit_data = {
+    'aerodrome': aero,
+    'spaceport': space,
+    'swarm': swarm,
+    'dns_vip_aerodrome': dns_aero,
+    'dns_vip_spaceport': dns_space,
+    'dns_vip_internal': dns_local,
+    'endpoints': endpoint_results
+}
+
+print(json.dumps(audit_data, indent=2))
+
--- a/scripts/inspect_npm_4_issues.py
+++ b/scripts/inspect_npm_4_issues.py
@@ -0,0 +1,29 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+def inspect_conf(pattern):
+    stdin, stdout, stderr = ssh.exec_command(f"sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) grep -rn '{pattern}' /data/nginx/proxy_host/")
+    lines = stdout.read().decode().strip().splitlines()
+    print(f"=== Matching {pattern} ===")
+    conf_files = set()
+    for l in lines:
+        print(l)
+        parts = l.split(":")
+        if parts: conf_files.add(parts[0])
+    for cf in conf_files:
+        stdin2, stdout2, stderr2 = ssh.exec_command(f"sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat {cf}")
+        print(f"\n--- {cf} ---")
+        print(stdout2.read().decode())
+
+inspect_conf('longhorn')
+inspect_conf('semaphore')
+inspect_conf('uptime')
+inspect_conf('sn-nas')
+inspect_conf('snarfnas')
+
+ssh.close()
+
--- a/scripts/inspect_provider_13.py
+++ b/scripts/inspect_provider_13.py
@@ -0,0 +1,26 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url):
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        return json.loads(resp.read().decode('utf-8'))
+
+print("=== Provider 13 (Homepage / snarfnet.net) ===")
+p13 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/13/")
+print(json.dumps(p13, indent=2))
+
--- a/scripts/inspect_proxy_providers.py
+++ b/scripts/inspect_proxy_providers.py
@@ -0,0 +1,34 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url):
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        return json.loads(resp.read().decode('utf-8'))
+
+print("=== Provider 24 (pihole1) ===")
+p24 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/24/")
+print(json.dumps(p24, indent=2))
+
+print("\n=== Provider 12 (pihole2) ===")
+p12 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/12/")
+print(json.dumps(p12, indent=2))
+
+print("\n=== Outposts ===")
+outposts = api_call("https://authentik.snarfnet.net/api/v3/outposts/instances/")
+print(json.dumps(outposts, indent=2))
+
--- a/scripts/inspect_semaphore.py
+++ b/scripts/inspect_semaphore.py
@@ -0,0 +1,49 @@
+import paramiko
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url):
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        return json.loads(resp.read().decode('utf-8'))
+
+print("=== Provider 10 in Authentik ===")
+try:
+    p10 = api_call("https://authentik.snarfnet.net/api/v3/providers/oauth2/10/")
+    print("OAuth2 Provider 10:")
+    print(json.dumps(p10, indent=2))
+except Exception as e:
+    print("Not OAuth2:", e)
+    try:
+        p10 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/10/")
+        print("Proxy Provider 10:")
+        print(json.dumps(p10, indent=2))
+    except Exception as e2:
+        print("Not Proxy either:", e2)
+
+# Check Semaphore deployment in Spaceport
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get deployment -n infrastructure semaphore -o yaml")
+print("\n=== Semaphore Deployment YAML ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/investigate_issues_1_2.py
+++ b/scripts/investigate_issues_1_2.py
@@ -0,0 +1,31 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+def check_issue1_and_2():
+    print("=== Checking Ingress on Aerodrome (Issue 1) ===")
+    ssh_aero = paramiko.SSHClient()
+    ssh_aero.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh_aero.connect('140.44.4.11', username='root', key_filename=key_path, timeout=10)
+
+    stdin, stdout, stderr = ssh_aero.exec_command("k3s kubectl get ingress -A -o wide")
+    print(stdout.read().decode())
+
+    stdin, stdout, stderr = ssh_aero.exec_command("k3s kubectl get ingress -n longhorn-system -o yaml 2>/dev/null || true")
+    print("Longhorn Ingress YAML on Aerodrome:")
+    print(stdout.read().decode())
+
+    ssh_aero.close()
+
+    print("\n=== Checking Semaphore on Spaceport (Issue 2) ===")
+    ssh_space = paramiko.SSHClient()
+    ssh_space.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh_space.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+    stdin, stdout, stderr = ssh_space.exec_command("k3s kubectl get ingress -A -o wide")
+    print(stdout.read().decode())
+
+    ssh_space.close()
+
+check_issue1_and_2()
+
--- a/scripts/read_11.py
+++ b/scripts/read_11.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/11.conf")
+print("=== /data/nginx/proxy_host/11.conf (sn-nas) ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/read_37.py
+++ b/scripts/read_37.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/37.conf")
+print("=== /data/nginx/proxy_host/37.conf ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/test_aero_uptime.py
+++ b/scripts/test_aero_uptime.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("curl -k -I -H 'Host: uptime.snarfnet.net' https://140.44.4.34:443/")
+print("Aerodrome Traefik uptime.snarfnet.net:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/test_authentik_flow_api.py
+++ b/scripts/test_authentik_flow_api.py
@@ -0,0 +1,21 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+url = "https://authentik.snarfnet.net/api/v3/flows/executor/default-authentication-flow/?client_id=H5MRodgRNAMuKv2fXsj3roxfO1f9ImOSRAN2Pi47&redirect_uri=https%3A%2F%2Fpihole1.snarfnet.net%2Foutpost.goauthentik.io%2Fcallback%3FX-authentik-auth-callback%3Dtrue&response_type=code&scope=ak_proxy+openid+entitlements+profile+email"
+
+req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        print("Status code:", resp.status)
+        data = resp.read().decode('utf-8')
+        print("API Response:", data[:500])
+except urllib.error.HTTPError as e:
+    print(f"HTTP Error {e.code}: {e.read().decode('utf-8')[:500]}")
+except Exception as e:
+    print("Exception:", e)
+
--- a/scripts/test_authorize_url.py
+++ b/scripts/test_authorize_url.py
@@ -0,0 +1,24 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+# Test authorize URL for pihole1
+url1 = "https://authentik.snarfnet.net/application/o/authorize/?client_id=H5MRodgRNAMuKv2fXsj3roxfO1f9ImOSRAN2Pi47&redirect_uri=https%3A%2F%2Fpihole1.snarfnet.net%2Foutpost.goauthentik.io%2Fcallback%3FX-authentik-auth-callback%3Dtrue&response_type=code&scope=ak_proxy+openid+entitlements+profile+email"
+
+req = urllib.request.Request(url1, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        print("Status code:", resp.status)
+        print("Final URL:", resp.url)
+        content = resp.read().decode('utf-8')
+        print("Content sample (first 500 chars):")
+        print(content[:500])
+except urllib.error.HTTPError as e:
+    print(f"HTTP Error {e.code}: {e.read().decode('utf-8')[:500]}")
+except Exception as e:
+    print("Exception:", e)
+
--- a/scripts/test_nas_from_npm.py
+++ b/scripts/test_nas_from_npm.py
@@ -0,0 +1,24 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+print("Curl 14.10.10.71:5001 (sn-nas):")
+stdin, stdout, stderr = ssh.exec_command("curl -k -I -m 5 https://14.10.10.71:5001/")
+print(stdout.read().decode())
+print(stderr.read().decode())
+
+print("Curl 14.10.10.72:5001 (snarfnas):")
+stdin, stdout, stderr = ssh.exec_command("curl -k -I -m 5 https://14.10.10.72:5001/")
+print(stdout.read().decode())
+print(stderr.read().decode())
+
+print("Curl 140.44.4.29:9000 (old authentik ip?):")
+stdin, stdout, stderr = ssh.exec_command("curl -I -m 3 http://140.44.4.29:9000/outpost.goauthentik.io/auth/nginx")
+print(stdout.read().decode())
+print(stderr.read().decode())
+
+ssh.close()
+
--- a/scripts/test_sem_correct_route.py
+++ b/scripts/test_sem_correct_route.py
@@ -0,0 +1,16 @@
+import urllib.request
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+url = "https://semaphore.snarfnet.net/auth/oidc/authentik/redirect"
+req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+try:
+    with urllib.request.urlopen(req, context=ctx) as resp:
+        print("Status:", resp.status)
+        print(resp.read().decode())
+except urllib.error.HTTPError as e:
+    print(f"HTTP {e.code}:", e.read().decode())
+
--- a/scripts/test_sem_redirect.py
+++ b/scripts/test_sem_redirect.py
@@ -0,0 +1,20 @@
+import urllib.request
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def test_url(url):
+    print(f"Testing {url}:")
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        with urllib.request.urlopen(req, context=ctx) as resp:
+            print("Status:", resp.status)
+            print(resp.read().decode())
+    except urllib.error.HTTPError as e:
+        print(f"HTTP {e.code}:", e.read().decode())
+
+test_url("https://semaphore.snarfnet.net/api/auth/oidc/authentik/redirect")
+test_url("https://semaphore.snarfnet.net/api/auth/oidc/authentik/redirect/")
+
--- a/scripts/test_spaceport_traefik.py
+++ b/scripts/test_spaceport_traefik.py
@@ -0,0 +1,24 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=key_path, timeout=10)
+
+# Check curl from spaceport host
+stdin, stdout, stderr = ssh.exec_command("curl -k -I https://authentik.snarfnet.net/outpost.goauthentik.io/auth/traefik")
+print("Curl from Spaceport host:")
+print(stdout.read().decode())
+
+# Check Traefik pod on spaceport
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl exec -n kube-system deployment/traefik -- curl -k -I https://authentik.snarfnet.net/outpost.goauthentik.io/auth/traefik 2>/dev/null || true")
+print("Curl from Traefik pod on Spaceport:")
+print(stdout.read().decode())
+
+# Check Traefik logs on spaceport
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl logs -n kube-system deployment/traefik --tail=30 | grep -E 'pihole|authentik|refused' || true")
+print("Traefik logs on Spaceport:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/test_syno_script.py
+++ b/scripts/test_syno_script.py
@@ -0,0 +1,21 @@
+import urllib.request
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+url = "https://sn-nas.snarfnet.net/webapi/entry.cgi?api=SYNO.Core.Desktop.SessionData&version=1&method=getjs&SynoToken=&v=1779933775"
+req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        print("Status:", resp.status)
+        print("Headers:", dict(resp.headers))
+        content = resp.read().decode('utf-8', errors='ignore')
+        print("Script length:", len(content))
+        print("Script preview:\n", content[:300])
+except urllib.error.HTTPError as e:
+    print(f"HTTP Error {e.code}:", e.read().decode('utf-8', errors='ignore')[:300])
+except Exception as e:
+    print("Error:", e)
+
--- a/scripts/test_the_4_issues.py
+++ b/scripts/test_the_4_issues.py
@@ -0,0 +1,28 @@
+import urllib.request
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def fetch(url):
+    print(f"=== Fetching {url} ===")
+    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
+    try:
+        with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+            print("Status:", resp.status)
+            print("Headers:", dict(resp.headers))
+            data = resp.read().decode('utf-8', errors='ignore')
+            print("Body (first 500 chars):")
+            print(data[:500])
+    except urllib.error.HTTPError as e:
+        print(f"HTTP Error {e.code}:", e.read().decode('utf-8', errors='ignore')[:300])
+    except Exception as e:
+        print("Error:", e)
+
+fetch("https://sn-nas.snarfnet.net")
+fetch("https://snarfnas.snarfnet.net")
+fetch("https://semaphore.snarfnet.net")
+fetch("https://uptime.snarfnet.net")
+fetch("https://longhorn.aerodrome.snarfnet.net")
+
--- a/scripts/test_wan_pihole.py
+++ b/scripts/test_wan_pihole.py
@@ -0,0 +1,36 @@
+import urllib.request
+import ssl
+import http.client
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def test_external(domain):
+    print(f"\n=== Testing External Request to {domain} ===")
+    url = f"https://{domain}/"
+    req = urllib.request.Request(
+        url,
+        headers={
+            'Host': domain,
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
+        }
+    )
+    
+    # We test connecting directly to WAN IP with SNI
+    conn = http.client.HTTPSConnection("174.106.252.96", 443, context=ctx, timeout=8)
+    conn.request("GET", "/", headers={
+        'Host': domain,
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+    })
+    resp = conn.getresponse()
+    print("Status code:", resp.status)
+    print("Location header:", resp.getheader('Location'))
+    print("Set-Cookie header:", resp.getheader('Set-Cookie'))
+    conn.close()
+
+test_external('pihole1.snarfnet.net')
+test_external('pihole2.snarfnet.net')
+
--- a/scripts/test_wan_pihole_sni.py
+++ b/scripts/test_wan_pihole_sni.py
@@ -0,0 +1,27 @@
+import socket
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def test_external(domain, ip="174.106.252.96"):
+    print(f"\n=== Testing External Request to {domain} via {ip} ===")
+    with socket.create_connection((ip, 443), timeout=5) as sock:
+        with ctx.wrap_socket(sock, server_hostname=domain) as ssock:
+            req = f"GET /admin/ HTTP/1.1\r\nHost: {domain}\r\nUser-Agent: Mozilla/5.0\r\nConnection: close\r\n\r\n"
+            ssock.sendall(req.encode())
+            resp = b""
+            while True:
+                chunk = ssock.recv(4096)
+                if not chunk: break
+                resp += chunk
+            headers = resp.split(b"\r\n\r\n")[0].decode('utf-8', errors='ignore')
+            print("Headers:")
+            for line in headers.splitlines():
+                if any(k in line.lower() for k in ['http/1.1', 'location', 'set-cookie', 'server', 'content-length']):
+                    print(" ", line)
+
+test_external('pihole1.snarfnet.net')
+test_external('pihole2.snarfnet.net')
+
--- a/scripts/update_p12.py
+++ b/scripts/update_p12.py
@@ -0,0 +1,38 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url, method="GET", payload=None):
+    data = json.dumps(payload).encode('utf-8') if payload else None
+    req = urllib.request.Request(url, headers=headers, data=data, method=method)
+    with urllib.request.urlopen(req, context=ctx, timeout=15) as resp:
+        return json.loads(resp.read().decode('utf-8'))
+
+payload = {
+    "mode": "forward_domain",
+    "external_host": "https://authentik.snarfnet.net",
+    "cookie_domain": "snarfnet.net",
+    "redirect_uris": [
+        {
+            "matching_mode": "strict",
+            "url": "https://authentik.snarfnet.net/outpost.goauthentik.io/callback?X-authentik-auth-callback=true"
+        }
+    ]
+}
+
+res12 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/12/", method="PATCH", payload=payload)
+print("Updated Provider 12:", res12.get('name'), "| Mode:", res12.get('mode'), "| Cookie Domain:", res12.get('cookie_domain'))
+
--- a/scripts/update_providers_forward_domain.py
+++ b/scripts/update_providers_forward_domain.py
@@ -0,0 +1,46 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url, method="GET", payload=None):
+    data = json.dumps(payload).encode('utf-8') if payload else None
+    req = urllib.request.Request(url, headers=headers, data=data, method=method)
+    try:
+        with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+            return json.loads(resp.read().decode('utf-8'))
+    except urllib.error.HTTPError as e:
+        print(f"API Error {e.code} on {method} {url}: {e.read().decode('utf-8')}")
+        return None
+
+# Update Provider 24 to forward_domain with cookie_domain snarfnet.net
+payload = {
+    "mode": "forward_domain",
+    "external_host": "https://authentik.snarfnet.net",
+    "cookie_domain": "snarfnet.net",
+    "redirect_uris": [
+        {
+            "matching_mode": "strict",
+            "url": "https://authentik.snarfnet.net/outpost.goauthentik.io/callback?X-authentik-auth-callback=true"
+        }
+    ]
+}
+
+res24 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/24/", method="PATCH", payload=payload)
+print("Updated Provider 24:", res24.get('name'), "| Mode:", res24.get('mode'), "| Cookie Domain:", res24.get('cookie_domain'))
+
+res12 = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/12/", method="PATCH", payload=payload)
+print("Updated Provider 12:", res12.get('name'), "| Mode:", res12.get('mode'), "| Cookie Domain:", res12.get('cookie_domain'))
+