feat: add cluster management and audit utility scripts

scripts/apply_36.py

@@ -0,0 +1,65 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n dns delete svc pihole-dns-vip pihole-web-vip --wait=false
+sleep 2
+cat << 'EOF' | k3s kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: pihole-dns-vip
+  namespace: dns
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 140.44.4.36
+    metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
+  labels:
+    app: pihole
+spec:
+  type: LoadBalancer
+  selector:
+    app: pihole
+  ports:
+  - name: dns-udp
+    port: 53
+    targetPort: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    targetPort: 53
+    protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pihole-web-vip
+  namespace: dns
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 140.44.4.36
+    metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
+  labels:
+    app: pihole
+spec:
+  type: LoadBalancer
+  selector:
+    app: pihole
+  ports:
+  - name: web-http
+    port: 80
+    targetPort: 80
+    protocol: TCP
+EOF
+sleep 3
+k3s kubectl -n dns get svc
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/apply_dns_updates.py

@@ -0,0 +1,38 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def apply_to_cluster(ip, name, manifest_path):
+    print(f"Connecting to {name} ({ip})...")
+    with open(manifest_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(ip, username="root", key_filename=key_path, timeout=10)
+    
+    stdin, stdout, stderr = c.exec_command("k3s kubectl apply -f -")
+    stdin.write(content)
+    stdin.channel.shutdown_write()
+    
+    out = stdout.read().decode('utf-8')
+    err = stderr.read().decode('utf-8')
+    print(f"[{name}] Apply Output:\n{out}")
+    if err.strip():
+        print(f"[{name}] Apply Error:\n{err}")
+    c.close()
+
+aero_path = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml"
+space_path = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml"
+
+apply_to_cluster("140.44.4.11", "Aerodrome", aero_path)
+time.sleep(2)
+apply_to_cluster("140.44.4.41", "Spaceport", space_path)
+print("Finished applying manifests to both clusters!")
+
+
+

scripts/apply_web_vips.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+import base64
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def apply_manifest(host, file_path):
+    with open(file_path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode('ascii')
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=5)
+    stdin, stdout, stderr = c.exec_command(f"echo '{b64}' | base64 -d | k3s kubectl apply -f -")
+    print(f"Apply on {host}:")
+    print(stdout.read().decode('utf-8'))
+    c.close()
+
+apply_manifest("140.44.4.11", r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml")
+apply_manifest("140.44.4.41", r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml")
+

scripts/audit_authentik.py

@@ -0,0 +1,30 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+def run_query(title, q):
+    cmd = f"""k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"{q}\\"" """
+    stdin, stdout, stderr = c.exec_command(cmd)
+    print(f"=== {title} ===")
+    out = stdout.read().decode('utf-8')
+    err = stderr.read().decode('utf-8')
+    if out.strip(): print(out.strip())
+    if err.strip(): print("ERR:", err.strip())
+    print()
+
+run_query("OUTPOST PROVIDERS", "SELECT outpost_id, provider_id FROM authentik_outposts_outpost_providers;")
+
+
+
+
+
+
+
+
+c.close()

scripts/check_aero_infra.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get deploy,svc,pod")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/check_aero_svc.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get svc")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/check_auth_links.py

@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT provider_id, application_id FROM authentik_core_application_backchannel_providers;\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Backchannel Providers:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT * FROM authentik_providers_proxy_proxyprovider;\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Proxy Providers:")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_auth_tables.py

@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT name, slug FROM authentik_core_application;\"'"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Apps:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT name, external_host FROM authentik_providers_proxy_proxyprovider;\"'"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Proxy Providers:")
+print(stdout.read().decode('utf-8'))
+
+cmd3 = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT name, client_id FROM authentik_providers_oauth2_oauth2provider;\"'"
+stdin, stdout, stderr = c.exec_command(cmd3)
+print("OAuth2/OIDC Providers:")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_authentik_apps.py

@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, authorization_flow_id FROM authentik_core_application;'"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Authentik Applications:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, external_host FROM authentik_providers_proxy_proxyprovider;'"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Authentik Proxy Providers:")
+print(stdout.read().decode('utf-8'))
+
+cmd3 = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, client_id FROM authentik_providers_oauth2_oauth2provider;'"
+stdin, stdout, stderr = c.exec_command(cmd3)
+print("Authentik OAuth2/OIDC Providers:")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_authentik_ingress.py

@@ -0,0 +1,19 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get middleware.traefik.io,middleware.traefik.containo.us -A 2>/dev/null")
+print("=== TRAEFIK MIDDLEWARES ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get ingress -A")
+print("=== ALL INGRESSES ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_dns_query.py

@@ -0,0 +1,27 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "dig @140.44.4.36 google.com"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("=== DIG FULL OUTPUT ===")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n dns logs deploy/pihole --tail=25"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("=== PIHOLE LOGS ===")
+print(stdout.read().decode('utf-8'))
+
+cmd3 = "k3s kubectl -n dns logs deploy/technitium-dns --tail=25"
+stdin, stdout, stderr = c.exec_command(cmd3)
+print("=== TECHNITIUM LOGS ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+

scripts/check_dns_status.py

@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns get pods,svc")
+print("=== PODS & SVCS ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns describe svc pihole-dns-vip | grep -E 'Events:|Message|Failed'")
+print("=== SERVICE EVENTS ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+

scripts/check_dns_svcs.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n dns get pods,svc -o wide"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("=== AERODROME DNS WORKLOADS ===")
+print(stdout.read().decode('utf-8'))
+
+c.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+stdin, stdout, stderr = c.exec_command(cmd)
+print("=== SPACEPORT DNS WORKLOADS ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_env_in_pod.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns exec deployment/technitium-dns -- env")
+print("ENV IN POD:")
+for line in stdout.read().decode('utf-8').splitlines():
+    if "DNS_SERVER" in line:
+        print(" ", line)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns exec deployment/technitium-dns -- ls -la /etc/dns/logs /etc/dns")
+print("FILES:")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_flows.py

@@ -0,0 +1,29 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+req = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/flows/instances/?designation=authorization",
+    headers={"Authorization": "Bearer SnarfNetPermanentAutomationToken2026"}
+)
+
+with urllib.request.urlopen(req, context=ctx, timeout=5) as resp:
+    data = json.loads(resp.read().decode('utf-8'))
+    print("Authorization flows:")
+    for f in data.get('results', []):
+        print(f"  {f.get('name')} (slug: {f.get('slug')}, pk: {f.get('pk')})")
+
+req3 = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/propertymappings/provider/scope/",
+    headers={"Authorization": "Bearer SnarfNetPermanentAutomationToken2026"}
+)
+with urllib.request.urlopen(req3, context=ctx, timeout=5) as resp:
+    data3 = json.loads(resp.read().decode('utf-8'))
+    print("Scope Property Mappings:")
+    for m in data3.get('results', []):
+        print(f"  {m.get('scope_name')}: {m.get('pk')}")
+

scripts/check_forwardauth.py

@@ -0,0 +1,13 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n longhorn-system get middleware authentik-forwardauth -o yaml")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/check_ingress_yaml.py

@@ -0,0 +1,23 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n monitoring get ingress uptime-ing -o yaml")
+print("=== UPTIME INGRESS ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n longhorn-system get ingress aerodrome-longhorn -o yaml")
+print("=== LONGHORN INGRESS ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get middleware -A")
+print("=== MIDDLEWARES ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_npm_hosts.py

@@ -4,12 +4,15 @@ import sys
 sys.stdout.reconfigure(encoding='utf-8')
 
 key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
-client = paramiko.SSHClient()
-client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=5)
 
-client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
-stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=npm) sqlite3 /data/database.sqlite "SELECT domain_names, forward_host, forward_port FROM proxy_host;"')
-print("NPM hosts:")
+cmd = """docker service ls"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("SERVICES:")
 print(stdout.read().decode('utf-8'))
-client.close()
 
+
+
+c.close()

scripts/check_oauth_cols.py

@@ -0,0 +1,17 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT name, client_id FROM authentik_providers_oauth2_oauth2provider;\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("OAuth2:")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/check_pg_env.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- env | grep POSTGRES"
+stdin, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/check_pid.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("cat /proc/2875586/cmdline | tr '\\0' ' '")
+print("Cmdline:", stdout.read().decode('utf-8'))
+c.close()
+

scripts/check_pihole_app.py

@@ -0,0 +1,21 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT * FROM authentik_core_application WHERE slug = \\\"pihole\\\";\"'"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Pi-hole App:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT * FROM authentik_providers_proxy_proxyprovider WHERE name ILIKE \\\"%pihole%\\\";\"'"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Pi-hole Proxy Provider:")
+print(stdout.read().decode('utf-8'))
+
+c.close()

scripts/check_pihole_url.py

@@ -0,0 +1,17 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT name, slug, meta_launch_url FROM authentik_core_application WHERE slug = 'pihole';\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Pi-hole App:")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/check_ps.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("ps aux | grep kubectl")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/check_psql_direct.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra exec authentik-postgresql-0 -i -- psql -U authentik -d authentik -c 'SELECT count(*) FROM authentik_core_application;'")
+print(stdout.read().decode('utf-8'))
+err = stderr.read().decode('utf-8')
+if err: print("ERR:", err)
+c.close()

scripts/check_sc.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get sc")
+print("StorageClasses:")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/check_space_infra.py

@@ -0,0 +1,13 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infrastructure get deploy,svc,pod | grep -E 'pihole|technitium'")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/check_space_pods.py

@@ -0,0 +1,19 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns get pods,svc -o wide")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns logs -l app=pihole --tail=15")
+print("=== PIHOLE LOGS ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+

scripts/check_technitium_auth.py

@@ -0,0 +1,19 @@
+import urllib.request
+import json
+
+passwords = ['snarfnetadmin', 'admin', 'password', 'SnarfNet2024!', 'SnarfNet2026!', 'root', 'snarf', 'administrator']
+
+for host in ['140.44.4.36', '140.44.4.65']:
+    print(f"Testing {host}:")
+    for pwd in passwords:
+        url = f"http://{host}:5380/api/user/login?user=admin&pass={pwd}"
+        try:
+            with urllib.request.urlopen(url, timeout=3) as resp:
+                data = json.loads(resp.read().decode('utf-8'))
+                if data.get('status') == 'ok':
+                    print(f"  SUCCESS! Password is '{pwd}', token: {data.get('token')}")
+                    break
+                else:
+                    print(f"  Failed '{pwd}': {data.get('errorMessage')}")
+        except Exception as e:
+            print(f"  Error on {pwd}: {e}")

scripts/create_dns_wildcard_secret.py

@@ -0,0 +1,47 @@
+import paramiko
+import sys
+import base64
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+# 1. Read npm-1 (*.snarfnet.net) from NPM on 140.44.4.71
+cNpm = paramiko.SSHClient()
+cNpm.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+cNpm.connect("140.44.4.71", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = cNpm.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /etc/letsencrypt/live/npm-1/fullchain.pem")
+fullchain = stdout.read()
+stdin, stdout, stderr = cNpm.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /etc/letsencrypt/live/npm-1/privkey.pem")
+privkey = stdout.read()
+cNpm.close()
+
+b64_crt = base64.b64encode(fullchain).decode('ascii')
+b64_key = base64.b64encode(privkey).decode('ascii')
+
+def apply_wildcard_secret(host, name):
+    print(f"Creating secret tls-snarfnet-wildcard on {name} ({host})...")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=5)
+    
+    cmd = f"""
+cat << 'EOF' | k3s kubectl apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: tls-snarfnet-wildcard
+  namespace: dns
+type: kubernetes.io/tls
+data:
+  tls.crt: {b64_crt}
+  tls.key: {b64_key}
+EOF
+"""
+    stdin, stdout, stderr = c.exec_command(cmd)
+    print(stdout.read().decode('utf-8'))
+    c.close()
+
+apply_wildcard_secret("140.44.4.11", "Aerodrome")
+apply_wildcard_secret("140.44.4.41", "Spaceport")

scripts/create_technitium_oidc.py

@@ -0,0 +1,105 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+provider_payload = {
+    "name": "Technitium DNS Server",
+    "authorization_flow": "38a86ef3-f43a-4756-8f54-ca008a8eef6e",
+    "invalidation_flow": "7a8145b8-2476-4636-9901-825f826df805",
+    "client_type": "confidential",
+    "client_id": "technitium-dns-oidc",
+    "client_secret": "SnarfNetTechnitiumOIDC2026!",
+    "redirect_uris": [
+        {"url": "https://technitium.snarfnet.net/sso/callback", "matching_mode": "strict"},
+        {"url": "https://dns1.snarfnet.net/sso/callback", "matching_mode": "strict"},
+        {"url": "https://dns2.snarfnet.net/sso/callback", "matching_mode": "strict"},
+        {"url": "https://technitium.snarfnet.net/", "matching_mode": "strict"},
+        {"url": "https://dns1.snarfnet.net/", "matching_mode": "strict"},
+        {"url": "https://dns2.snarfnet.net/", "matching_mode": "strict"},
+        {"url": "https://.*\\.snarfnet\\.net/sso/callback", "matching_mode": "regex"}
+    ],
+    "signing_key": "6d48ba88-4b51-428f-903c-93334988a941",
+    "property_mappings": [
+        "f521356c-080d-4bd6-b19a-68ab159a5aef", # openid
+        "8395b909-8e8d-4fc1-a620-5592e10df8fe", # email
+        "85b89b43-92d8-425c-b9cc-8e00038cad37", # profile
+        "d2248142-6be2-4833-94f8-e02f5d293b02"  # groups
+    ],
+    "sub_mode": "hashed_user_id",
+    "issuer_mode": "per_provider",
+    "include_claims_in_id_token": True
+}
+
+print("1. Creating OAuth2 Provider in Authentik...")
+req = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/providers/oauth2/",
+    headers=headers,
+    data=json.dumps(provider_payload).encode('utf-8'),
+    method="POST"
+)
+
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        prov_data = json.loads(resp.read().decode('utf-8'))
+        prov_pk = prov_data.get('pk')
+        print(f"Provider created successfully! PK: {prov_pk}")
+except urllib.error.HTTPError as e:
+    err_body = e.read().decode('utf-8')
+    print(f"Provider creation failed ({e.code}): {err_body}")
+    # Check if it already exists
+    req_check = urllib.request.Request(
+        "https://authentik.snarfnet.net/api/v3/providers/oauth2/?search=Technitium",
+        headers=headers
+    )
+    with urllib.request.urlopen(req_check, context=ctx, timeout=8) as resp_c:
+        existing = json.loads(resp_c.read().decode('utf-8'))
+        if existing.get('results'):
+            prov_pk = existing['results'][0]['pk']
+            print(f"Using existing provider PK: {prov_pk}")
+        else:
+            sys.exit(1)
+
+print("2. Creating Application in Authentik...")
+app_payload = {
+    "name": "Technitium DNS Server",
+    "slug": "technitium",
+    "provider": prov_pk,
+    "group": "Infrastructure & Hypervisors",
+    "meta_launch_url": "https://technitium.snarfnet.net",
+    "meta_description": "Authoritative & Recursive DNS Server with Ad Blocking and Split-Horizon",
+    "meta_icon": "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/technitium.png",
+    "open_in_new_tab": True
+}
+
+req_app = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/core/applications/",
+    headers=headers,
+    data=json.dumps(app_payload).encode('utf-8'),
+    method="POST"
+)
+
+try:
+    with urllib.request.urlopen(req_app, context=ctx, timeout=8) as resp:
+        app_data = json.loads(resp.read().decode('utf-8'))
+        print(f"Application created successfully! Slug: {app_data.get('slug')}")
+except urllib.error.HTTPError as e:
+    err_body = e.read().decode('utf-8')
+    print(f"Application creation ({e.code}): {err_body}")
+
+print("\nOIDC Setup Details:")
+print(f"  Issuer / Metadata: https://authentik.snarfnet.net/application/o/technitium/.well-known/openid-configuration")
+print(f"  Client ID: technitium-dns-oidc")
+print(f"  Client Secret: SnarfNetTechnitiumOIDC2026!")

scripts/delete_nowait.py

@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+print("--- Aerodrome ---")
+c1 = paramiko.SSHClient()
+c1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c1.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+stdin, stdout, stderr = c1.exec_command("k3s kubectl -n infra delete svc technitium-npm-vip-service --wait=false")
+print(stdout.read().decode('utf-8'))
+c1.close()
+
+print("--- Spaceport ---")
+c2 = paramiko.SSHClient()
+c2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c2.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+cmd = "k3s kubectl -n infrastructure delete deploy pihole technitium-dns svc pihole-dns-tcp pihole-dns-udp pihole-web technitium-service --wait=false --ignore-not-found=true"
+stdin, stdout, stderr = c2.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c2.close()
+

scripts/delete_space_svc.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infrastructure delete svc pihole-dns-tcp pihole-dns-udp pihole-web technitium-service --wait=false --ignore-not-found=true"
+stdin, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/deploy_aerodrome_dns.py

@@ -0,0 +1,41 @@
+import paramiko
+import sys
+import base64
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+manifest_path = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml"
+
+with open(manifest_path, "rb") as f:
+    b64_manifest = base64.b64encode(f.read()).decode('ascii')
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+cmd = f"""
+echo '{b64_manifest}' | base64 -d | k3s kubectl apply -f -
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Apply result:")
+print(stdout.read().decode('utf-8'))
+err = stderr.read().decode('utf-8')
+if err: print("Stderr:", err)
+
+# Poll for pods and PVCs
+for i in range(12):
+    time.sleep(5)
+    stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns get pvc,deploy,svc,pod -o wide")
+    out = stdout.read().decode('utf-8')
+    print(f"\n--- Check {i+1} ---")
+    print(out)
+    if "Running" in out and "pihole" in out and "technitium-dns" in out:
+        # Check if both are Running 1/1
+        if "1/1     Running" in out:
+            print("Pods are running!")
+            break
+
+c.close()
+

scripts/deploy_spaceport_dns.py

@@ -0,0 +1,39 @@
+import paramiko
+import sys
+import base64
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+manifest_path = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml"
+
+with open(manifest_path, "rb") as f:
+    b64_manifest = base64.b64encode(f.read()).decode('ascii')
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+
+cmd = f"""
+echo '{b64_manifest}' | base64 -d | k3s kubectl apply -f -
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Spaceport Apply result:")
+print(stdout.read().decode('utf-8'))
+err = stderr.read().decode('utf-8')
+if err: print("Stderr:", err)
+
+# Poll for pods and services
+for i in range(12):
+    time.sleep(5)
+    stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns get pvc,deploy,svc,pod -o wide")
+    out = stdout.read().decode('utf-8')
+    print(f"\n--- Check {i+1} ---")
+    print(out)
+    if "Running" in out and "140.44.4.65" in out:
+        print("Spaceport DNS stack is running and VIP is allocated!")
+        break
+
+c.close()
+

scripts/describe_dns_svc.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns describe svc pihole-dns-vip")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/find_pass.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+import re
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns exec deployment/technitium-dns -- cat /opt/technitium/dns/DnsServerCore.dll")
+dll = stdout.read()
+
+matches_u16 = set(re.findall(rb'(?:[A-Z0-9_]\x00){4,}', dll))
+for m in sorted(matches_u16):
+    s = m.decode('utf-16le')
+    if 'DNS' in s or 'PASS' in s or 'ADMIN' in s or 'AUTH' in s:
+        print("U16:", s)
+
+
+c.close()

scripts/finish_phase1_prune.py

@@ -0,0 +1,29 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+# 1. Aerodrome: delete technitium-npm-vip-service
+cAero = paramiko.SSHClient()
+cAero.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+cAero.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+stdin, stdout, stderr = cAero.exec_command("k3s kubectl -n infra delete svc technitium-npm-vip-service --ignore-not-found=true")
+print("Aerodrome delete svc:", stdout.read().decode('utf-8'))
+cAero.close()
+
+# 2. Spaceport: delete legacy resources in infrastructure
+cSpace = paramiko.SSHClient()
+cSpace.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+cSpace.connect("140.44.4.41", username="root", key_filename=key_path, timeout=5)
+cmd = """
+k3s kubectl -n infrastructure delete deploy pihole technitium-dns --ignore-not-found=true --grace-period=0 --force
+k3s kubectl -n infrastructure delete svc pihole-dns-tcp pihole-dns-udp pihole-web technitium-service --ignore-not-found=true
+k3s kubectl -n infrastructure delete pvc pihole technitium-longhorn-pvc --ignore-not-found=true
+"""
+stdin, stdout, stderr = cSpace.exec_command(cmd)
+print("Spaceport delete results:")
+print(stdout.read().decode('utf-8'))
+cSpace.close()
+

scripts/list_lbs.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get svc -A --field-selector spec.type=LoadBalancer -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,IP:.status.loadBalancer.ingress[0].ip")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/list_oauth2.py

@@ -0,0 +1,17 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT id, name, client_id, redirect_uris FROM authentik_providers_oauth2_oauth2provider;\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("OAuth2 Providers:")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/patch_svc_finalizer.py

@@ -0,0 +1,18 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get svc technitium-npm-vip-service -o jsonpath='{.metadata.deletionTimestamp}{\"\\n\"}{.metadata.finalizers}'")
+print(stdout.read().decode('utf-8'))
+
+# Remove finalizer if present
+c.exec_command("k3s kubectl -n infra patch svc technitium-npm-vip-service -p '{\"metadata\":{\"finalizers\":[]}}' --type=merge")
+
+c.close()
+

scripts/phase1_cleanup_stubs.py

@@ -0,0 +1,64 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def run_commands(host, ns, resources, desc):
+    print(f"\n=======================================================")
+    print(f"[{desc}] Cleaning up legacy DNS stubs on {host} ({ns})...")
+    print(f"=======================================================")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=10)
+    
+    for r in resources:
+        cmd = f"k3s kubectl -n {ns} delete {r} --ignore-not-found=true"
+        print(f">>> {cmd}")
+        stdin, stdout, stderr = c.exec_command(cmd)
+        out = stdout.read().decode('utf-8').strip()
+        err = stderr.read().decode('utf-8').strip()
+        if out: print("   ", out)
+        if err: print("   ERR:", err)
+        
+    print("\n--- Verifying remaining services and deployments ---")
+    stdin, stdout, stderr = c.exec_command(f"k3s kubectl -n {ns} get deploy,svc,pvc | grep -E 'pihole|technitium'")
+    res = stdout.read().decode('utf-8').strip()
+    if res:
+        print("Lingering items:\n", res)
+    else:
+        print("Clean! No lingering pihole or technitium resources found.")
+        
+    c.close()
+
+# 1. Aerodrome (ns: infra)
+aero_resources = [
+    "deployment pihole",
+    "deployment technitium-dns",
+    "service pihole-dns-tcp",
+    "service pihole-dns-udp",
+    "service pihole-web",
+    "service technitium-npm-vip-service",
+    "pvc pihole",
+    "pvc technitium-longhorn-pvc",
+    "configmap pihole-custom-dns",
+    "configmap technitium-config"
+]
+run_commands("140.44.4.11", "infra", aero_resources, "Aerodrome")
+
+# 2. Spaceport (ns: infrastructure)
+space_resources = [
+    "deployment pihole",
+    "deployment technitium-dns",
+    "service pihole-dns-tcp",
+    "service pihole-dns-udp",
+    "service pihole-web",
+    "service technitium-service",
+    "pvc pihole",
+    "pvc technitium-longhorn-pvc",
+    "configmap pihole-custom-dns",
+    "configmap technitium-config"
+]
+run_commands("140.44.4.41", "infrastructure", space_resources, "Spaceport")
+

scripts/reapply_aero_svc.py

@@ -0,0 +1,24 @@
+import paramiko
+import sys
+import base64
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+manifest_path = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml"
+
+with open(manifest_path, "rb") as f:
+    b64_manifest = base64.b64encode(f.read()).decode('ascii')
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = f"""
+echo '{b64_manifest}' | base64 -d | k3s kubectl apply -f -
+k3s kubectl -n dns get svc
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/recreate_aero_svc.py

@@ -0,0 +1,66 @@
+import paramiko
+import sys
+import base64
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n dns delete svc pihole-dns-vip pihole-web-vip --wait=false
+sleep 2
+cat << 'EOF' | k3s kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: pihole-dns-vip
+  namespace: dns
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 140.44.4.32
+    metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
+  labels:
+    app: pihole
+spec:
+  type: LoadBalancer
+  selector:
+    app: pihole
+  ports:
+  - name: dns-udp
+    port: 53
+    targetPort: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    targetPort: 53
+    protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pihole-web-vip
+  namespace: dns
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 140.44.4.32
+    metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
+  labels:
+    app: pihole
+spec:
+  type: LoadBalancer
+  selector:
+    app: pihole
+  ports:
+  - name: web-http
+    port: 80
+    targetPort: 80
+    protocol: TCP
+EOF
+sleep 3
+k3s kubectl -n dns get svc
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/repair_technitium_auth.py

@@ -0,0 +1,46 @@
+import paramiko
+import time
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+repair_yaml = """apiVersion: v1
+kind: Pod
+metadata:
+  name: dns-repair
+  namespace: dns
+spec:
+  containers:
+  - name: repair
+    image: busybox
+    command: ["sh", "-c", "ls -la /etc/dns && rm -f /etc/dns/auth.config* && echo AFTER REMOVAL: && ls -la /etc/dns"]
+    volumeMounts:
+    - name: data
+      mountPath: /etc/dns
+  volumes:
+  - name: data
+    persistentVolumeClaim:
+      claimName: technitium-config-pvc
+  restartPolicy: Never
+"""
+
+stdin, stdout, stderr = c.exec_command("cat << 'EOF' | k3s kubectl apply -f -\n" + repair_yaml + "\nEOF\n")
+print("Apply:", stdout.read().decode('utf-8'))
+time.sleep(8)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns logs pod/dns-repair")
+print("Logs:\n", stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns delete pod dns-repair --wait=false")
+print("Delete:", stdout.read().decode('utf-8'))
+
+# Now scale technitium back up!
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns scale deployment/technitium-dns --replicas=1")
+print("Scale up:", stdout.read().decode('utf-8'))
+
+c.close()

scripts/set_pihole_password.py

@@ -0,0 +1,23 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def set_pihole_pw(host, name):
+    print(f"\n=== Setting Pi-hole password on {name} ({host}) ===")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=5)
+    
+    # Try setting password via CLI
+    stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns exec deploy/pihole -- pihole -a -p snarfnetadmin || k3s kubectl -n dns exec deploy/pihole -- pihole setpassword snarfnetadmin")
+    print(stdout.read().decode('utf-8'))
+    err = stderr.read().decode('utf-8')
+    if err: print("Stderr:", err)
+    c.close()
+
+set_pihole_pw("140.44.4.11", "Aerodrome")
+set_pihole_pw("140.44.4.41", "Spaceport")
+

scripts/set_pihole_password_v6.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def set_pihole_pw(host, name):
+    print(f"\n=== Setting Pi-hole password on {name} ({host}) ===")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=5)
+    
+    stdin, stdout, stderr = c.exec_command("k3s kubectl -n dns exec deploy/pihole -- pihole setpassword 'snarfnetadmin'")
+    print(stdout.read().decode('utf-8'))
+    err = stderr.read().decode('utf-8')
+    if err: print("Stderr:", err)
+    c.close()
+
+set_pihole_pw("140.44.4.11", "Aerodrome")
+set_pihole_pw("140.44.4.41", "Spaceport")
+

scripts/setup_pihole_authentik.py

@@ -0,0 +1,91 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+# 1. Update Provider 12 (Pi-hole 2)
+p12_payload = {
+    "name": "Pi-hole 2 (Spaceport)",
+    "authorization_flow": "38a86ef3-f43a-4756-8f54-ca008a8eef6e",
+    "mode": "forward_single",
+    "external_host": "https://pihole2.snarfnet.net"
+}
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/providers/proxy/12/", headers=headers, data=json.dumps(p12_payload).encode('utf-8'), method="PATCH")
+with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+    print("Updated Provider 12 (Pi-hole 2) successfully!")
+
+# 2. Check if Pi-hole 1 provider exists, if not create it
+req_search = urllib.request.Request("https://authentik.snarfnet.net/api/v3/providers/proxy/?search=Pi-hole+1", headers=headers)
+with urllib.request.urlopen(req_search, context=ctx, timeout=8) as resp:
+    res = json.loads(resp.read().decode('utf-8'))
+    if res.get('results'):
+        p1_pk = res['results'][0]['pk']
+        print(f"Pi-hole 1 Provider already exists: PK {p1_pk}")
+    else:
+        p1_payload = {
+            "name": "Pi-hole 1 (Aerodrome)",
+            "authorization_flow": "38a86ef3-f43a-4756-8f54-ca008a8eef6e",
+            "invalidation_flow": "7a8145b8-2476-4636-9901-825f826df805",
+            "mode": "forward_single",
+            "external_host": "https://pihole1.snarfnet.net"
+        }
+        req_create = urllib.request.Request("https://authentik.snarfnet.net/api/v3/providers/proxy/", headers=headers, data=json.dumps(p1_payload).encode('utf-8'), method="POST")
+        try:
+            with urllib.request.urlopen(req_create, context=ctx, timeout=8) as resp_c:
+                p1_pk = json.loads(resp_c.read().decode('utf-8'))['pk']
+                print(f"Created Pi-hole 1 Provider: PK {p1_pk}")
+        except urllib.error.HTTPError as e:
+            print("Create error:", e.code, e.read().decode('utf-8'))
+            sys.exit(1)
+
+# 3. Add p1_pk to Embedded Outpost providers
+req_outpost = urllib.request.Request("https://authentik.snarfnet.net/api/v3/outposts/instances/e68edb7b-4996-438d-9747-e1591b173a66/", headers=headers)
+with urllib.request.urlopen(req_outpost, context=ctx, timeout=8) as resp:
+    outpost = json.loads(resp.read().decode('utf-8'))
+    current_providers = outpost.get('providers', [])
+    print("Current Outpost providers:", current_providers)
+    if p1_pk not in current_providers:
+        current_providers.append(p1_pk)
+        patch_outpost = urllib.request.Request(
+            "https://authentik.snarfnet.net/api/v3/outposts/instances/e68edb7b-4996-438d-9747-e1591b173a66/",
+            headers=headers,
+            data=json.dumps({"providers": current_providers}).encode('utf-8'),
+            method="PATCH"
+        )
+        with urllib.request.urlopen(patch_outpost, context=ctx, timeout=8) as resp_p:
+            print("Outpost updated with new provider!")
+    else:
+        print("Provider already in outpost.")
+
+# 4. Check/Create Application for Pi-hole 1
+req_app_search = urllib.request.Request("https://authentik.snarfnet.net/api/v3/core/applications/?search=Pi-hole+1", headers=headers)
+with urllib.request.urlopen(req_app_search, context=ctx, timeout=8) as resp:
+    res_app = json.loads(resp.read().decode('utf-8'))
+    if not res_app.get('results'):
+        app_payload = {
+            "name": "Pi-hole 1 (Aerodrome)",
+            "slug": "pihole1",
+            "provider": p1_pk,
+            "group": "Infrastructure & Hypervisors",
+            "meta_launch_url": "https://pihole1.snarfnet.net",
+            "meta_description": "Network-wide Ad and Tracker Blocking (Aerodrome Cluster)",
+            "meta_icon": "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/pi-hole.png",
+            "open_in_new_tab": True
+        }
+        req_app_c = urllib.request.Request("https://authentik.snarfnet.net/api/v3/core/applications/", headers=headers, data=json.dumps(app_payload).encode('utf-8'), method="POST")
+        with urllib.request.urlopen(req_app_c, context=ctx, timeout=8) as resp_ac:
+            print("Created Application Pi-hole 1!")
+    else:
+        print("Application Pi-hole 1 already exists.")

scripts/test_adblock.py

@@ -0,0 +1,28 @@
+import socket
+import struct
+
+def test_ad_block(server_ip, domain="pagead2.googlesyndication.com"):
+    header = struct.pack(">HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0)
+    qname = b"".join(bytes([len(p)]) + p.encode("ascii") for p in domain.split(".")) + b"\x00"
+    qtype_qclass = struct.pack(">HH", 1, 1)
+    packet = header + qname + qtype_qclass
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(3.0)
+    try:
+        sock.sendto(packet, (server_ip, 53))
+        resp, _ = sock.recvfrom(1024)
+        # Parse answer IP (last 4 bytes of standard A record answer)
+        if len(resp) >= 32:
+            ip = ".".join(str(b) for b in resp[-4:])
+            print(f"Server {server_ip} returned for {domain} -> {ip}")
+            if ip in ["0.0.0.0", "127.0.0.1"]:
+                print(f"  [VERIFIED] Ad domain {domain} was BLOCKED (0.0.0.0)!")
+    except Exception as e:
+        print(f"Error querying {server_ip}: {e}")
+    finally:
+        sock.close()
+
+test_ad_block("140.44.4.36")
+test_ad_block("140.44.4.65")
+

scripts/test_aero_dns.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "dig @140.44.4.36 google.com +short"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Querying Aerodrome Primary DNS (140.44.4.36) for google.com:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "curl -s -I http://140.44.4.36/admin/ | head -n 5"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Pi-hole Web Console HTTP response:")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+

scripts/test_api_auth.py

@@ -0,0 +1,17 @@
+import urllib.request
+import json
+
+def test_api_auth(ip):
+    url = f"http://{ip}/api/auth"
+    data = json.dumps({"password": "snarfnetadmin"}).encode('utf-8')
+    req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(req) as resp:
+            body = json.loads(resp.read().decode('utf-8'))
+            print(f"Server {ip}: Auth response ->", body)
+    except Exception as e:
+        print(f"Server {ip}: Auth failed -> {e}")
+
+test_api_auth("140.44.4.36")
+test_api_auth("140.44.4.65")
+

scripts/test_authentik_api.py

@@ -0,0 +1,22 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+req = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/core/applications/",
+    headers={"Authorization": "Bearer SnarfNetPermanentAutomationToken2026"}
+)
+
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=5) as resp:
+        data = json.loads(resp.read().decode('utf-8'))
+        print("Authentik API status:", resp.status)
+        print("Total applications:", data.get('pagination', {}).get('count'))
+        for app in data.get('results', [])[:5]:
+            print(f"  App: {app.get('name')} ({app.get('slug')})")
+except Exception as e:
+    print("API Error:", e)

scripts/test_both_vips.py

@@ -0,0 +1,27 @@
+import socket
+import struct
+
+def test_dns(server_ip, domain="google.com"):
+    header = struct.pack(">HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0)
+    qname = b"".join(bytes([len(p)]) + p.encode("ascii") for p in domain.split(".")) + b"\x00"
+    qtype_qclass = struct.pack(">HH", 1, 1)
+    packet = header + qname + qtype_qclass
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(4.0)
+    try:
+        sock.sendto(packet, (server_ip, 53))
+        resp, _ = sock.recvfrom(1024)
+        ancount = struct.unpack(">H", resp[6:8])[0]
+        print(f"SUCCESS: {server_ip} returned {len(resp)} bytes with {ancount} answers for {domain}!")
+    except Exception as e:
+        print(f"FAILED to query {server_ip}: {e}")
+    finally:
+        sock.close()
+
+print("Testing Aerodrome Primary VIP (140.44.4.36):")
+test_dns("140.44.4.36", "google.com")
+
+print("\nTesting Spaceport Secondary VIP (140.44.4.65):")
+test_dns("140.44.4.65", "google.com")
+

scripts/test_dns_packet.py

@@ -0,0 +1,27 @@
+import socket
+import struct
+
+def test_dns(server_ip, domain="google.com"):
+    # Build simple DNS query packet
+    # Transaction ID: 0x1234, Flags: 0x0100 (standard query, recursion desired)
+    header = struct.pack(">HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0)
+    qname = b"".join(bytes([len(p)]) + p.encode("ascii") for p in domain.split(".")) + b"\x00"
+    qtype_qclass = struct.pack(">HH", 1, 1) # A record, IN class
+    packet = header + qname + qtype_qclass
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(3.0)
+    try:
+        sock.sendto(packet, (server_ip, 53))
+        resp, _ = sock.recvfrom(1024)
+        print(f"SUCCESS: Received {len(resp)} bytes DNS response from {server_ip} for {domain}!")
+        # Check answer count
+        ancount = struct.unpack(">H", resp[6:8])[0]
+        print(f"Answer count: {ancount}")
+    except Exception as e:
+        print(f"FAILED to query {server_ip}: {e}")
+    finally:
+        sock.close()
+
+test_dns("140.44.4.36", "google.com")
+

scripts/test_forwardauth_endpoint.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("curl -s -I http://authentik-server.infra.svc.cluster.local/outpost.goauthentik.io/auth/traefik")
+print("Curl authentik forwardauth inside Aerodrome:")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/test_forwardauth_ip.py

@@ -0,0 +1,13 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("curl -s -I http://10.43.250.197/outpost.goauthentik.io/auth/traefik")
+print(stdout.read().decode('utf-8'))
+c.close()

scripts/test_nslookup.py

@@ -0,0 +1,28 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "nslookup google.com 140.44.4.36"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("=== NSLOOKUP ===")
+print("STDOUT:", stdout.read().decode('utf-8'))
+print("STDERR:", stderr.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n dns logs -l app=pihole --tail=20"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("=== PIHOLE LOGS ===")
+print(stdout.read().decode('utf-8'))
+
+cmd3 = "k3s kubectl -n dns logs -l app=technitium-dns --tail=20"
+stdin, stdout, stderr = c.exec_command(cmd3)
+print("=== TECHNITIUM LOGS ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+

scripts/test_pihole_login.py

@@ -0,0 +1,31 @@
+import urllib.request
+import urllib.parse
+import http.cookiejar
+
+def test_login(ip):
+    cj = http.cookiejar.CookieJar()
+    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
+    
+    # 1. Fetch login page to get CSRF/session
+    login_url = f"http://{ip}/admin/login"
+    req = urllib.request.Request(login_url, headers={"User-Agent": "Mozilla/5.0"})
+    with opener.open(req) as resp:
+        html = resp.read().decode('utf-8')
+        
+    # 2. Submit password
+    post_data = urllib.parse.urlencode({"pw": "snarfnetadmin", "login": "Log in (uses cookie)"}).encode('ascii')
+    req_post = urllib.request.Request(f"http://{ip}/admin/login", data=post_data, headers={"User-Agent": "Mozilla/5.0", "Referer": login_url})
+    
+    try:
+        with opener.open(req_post) as resp:
+            content = resp.read().decode('utf-8')
+            if "Wrong password" in content:
+                print(f"Server {ip}: WRONG PASSWORD!")
+            else:
+                print(f"Server {ip}: LOGIN SUCCESSFUL! Redirected to {resp.geturl()}")
+    except Exception as e:
+        print(f"Server {ip}: Exception during login: {e}")
+
+test_login("140.44.4.36")
+test_login("140.44.4.65")
+

scripts/test_technitium_login.py

@@ -0,0 +1,11 @@
+import urllib.request
+import json
+
+for host in ['140.44.4.36', '140.44.4.65']:
+    url = f"http://{host}:5380/api/user/login?user=admin&pass=snarfnetadmin"
+    try:
+        with urllib.request.urlopen(url, timeout=5) as resp:
+            data = json.loads(resp.read().decode('utf-8'))
+            print(f"{host}: status = {data.get('status')}, token = {data.get('token')}, msg = {data.get('errorMessage')}")
+    except Exception as e:
+        print(f"{host}: error = {e}")

scripts/test_web_consoles.py

@@ -0,0 +1,18 @@
+import urllib.request
+
+for ip in ["140.44.4.36", "140.44.4.65"]:
+    url = f"http://{ip}:5380/"
+    try:
+        req = urllib.request.urlopen(url, timeout=3)
+        print(f"Technitium Web UI on {url} -> HTTP {req.status}")
+    except Exception as e:
+        print(f"Error on {url}: {e}")
+
+for ip in ["140.44.4.36", "140.44.4.65"]:
+    url = f"http://{ip}/admin/"
+    try:
+        req = urllib.request.urlopen(url, timeout=3)
+        print(f"Pi-hole Web UI on {url} -> HTTP {req.status}")
+    except Exception as e:
+        print(f"Error on {url}: {e}")
+

scripts/update_pihole_provider.py

@@ -0,0 +1,49 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+# Fetch provider 12
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/providers/proxy/12/", headers=headers)
+with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+    data = json.loads(resp.read().decode('utf-8'))
+    print("Current Provider 12 external_host:", data.get('external_host'))
+    print("Current redirect URIs:", data.get('redirect_uris'))
+
+# Update redirect URIs
+redirect_uris = [
+    {"url": "https://pihole.snarfnet.net/outpost.goauthentik.io/callback?X-authentik-auth-callback=true", "matching_mode": "strict"},
+    {"url": "https://pihole.snarfnet.net?X-authentik-auth-callback=true", "matching_mode": "strict"},
+    {"url": "https://pihole1.snarfnet.net/outpost.goauthentik.io/callback?X-authentik-auth-callback=true", "matching_mode": "strict"},
+    {"url": "https://pihole1.snarfnet.net?X-authentik-auth-callback=true", "matching_mode": "strict"},
+    {"url": "https://pihole2.snarfnet.net/outpost.goauthentik.io/callback?X-authentik-auth-callback=true", "matching_mode": "strict"},
+    {"url": "https://pihole2.snarfnet.net?X-authentik-auth-callback=true", "matching_mode": "strict"},
+    {"url": "https://pihole.*\\.snarfnet\\.net.*", "matching_mode": "regex"}
+]
+
+patch_payload = {
+    "redirect_uris": redirect_uris
+}
+
+req_patch = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/providers/proxy/12/",
+    headers=headers,
+    data=json.dumps(patch_payload).encode('utf-8'),
+    method="PATCH"
+)
+
+try:
+    with urllib.request.urlopen(req_patch, context=ctx, timeout=8) as resp:
+        updated = json.loads(resp.read().decode('utf-8'))
+        print("Updated Provider 12 successfully!")
+except urllib.error.HTTPError as e:
+    print("PATCH error:", e.code, e.read().decode('utf-8'))

scripts/verify_aero_clean.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get svc technitium-npm-vip-service")
+print("STDOUT:", stdout.read().decode('utf-8'))
+print("STDERR:", stderr.read().decode('utf-8'))
+c.close()
+

scripts/verify_phase1_clean.py

@@ -0,0 +1,23 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def check(host, ns, name):
+    print(f"=== {name} ({ns}) ===")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=5)
+    stdin, stdout, stderr = c.exec_command(f"k3s kubectl -n {ns} get deploy,svc,pod | grep -E 'pihole|technitium' || true")
+    out = stdout.read().decode('utf-8').strip()
+    if out:
+        print(out)
+    else:
+        print("100% CLEAN - No lingering resources!")
+    c.close()
+
+check("140.44.4.11", "infra", "Aerodrome")
+check("140.44.4.41", "infrastructure", "Spaceport")
+