feat: add diagnostic and maintenance scripts for Pi-hole, Technitium, and Nginx Proxy Manager

docs/homelab_topology.md

@@ -99,12 +99,12 @@ All Kubernetes resources are hosted on the **Compute VLAN (VLAN 20 — `140.44.4
 
 ---
 
-### AI Stack (Raspberry Pis)
+### AI Stack (Raspberry Pis with Hailo NPU)
-*   **Infrastructure role:** Edge AI inference and local web interface.
+*   **Infrastructure role:** Edge AI inference backend and Open WebUI frontend.
 *   **SSH Default User:** `snarf`
 *   **IP Allocation:**
-    *   `snnpud4lnxops01` $\rightarrow$ `140.44.4.7` (Raspberry Pi with AI Hat 2, arm64, Ready)
+    *   `snnpud4lnxops01` $\rightarrow$ `140.44.4.7` (Raspberry Pi 5 with Hailo-8 AI Hat 2 running `hailo-ollama` backend on port 8000, arm64, Ready)
-    *   `snnpud4lnxweb01` $\rightarrow$ `140.44.4.8` (Raspberry Pi running Open WebUI, arm64, Ready)
+    *   `snnpud4lnxweb01` $\rightarrow$ `140.44.4.8` (Raspberry Pi running Open WebUI frontend on port 8080, arm64, Ready)
 
 ---
 
@@ -136,8 +136,7 @@ This table records how subdomains on `*.snarfnet.net` route to internal IP addre
 | `pihole.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8041 | Legacy Swarm Pi-hole 1 |
 | `pihole2.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8042 | Legacy Swarm Pi-hole 2 |
 | `shipyard.snarfnet.net` | HTTPS | `140.44.4.71` (Swarm 01) | 9443 | Portainer Server Console |
-| `shipyardcontrol.snarfnet.net` | HTTPS | `140.44.4.7` (AI Node) | 443 | Turing Pi Management / Hardware Panel |
+| `ai.snarfnet.net` | HTTP | `140.44.4.8` (`snnpud4lnxweb01`) | 8080 | Open WebUI Inference Interface |
-| `ai.snarfnet.net` | HTTP | `140.44.4.8` (AI Web Node) | 8080 | Open WebUI Inference Interface |
 | `aerodrome.snarfnet.net` | HTTPS | `140.44.4.34` (K3s VIP) | 443 | Rancher Management Console |
 | `grafana.snarfnet.net` | HTTP | `140.44.4.32` (K3s VIP) | 88 | Grafana Monitoring Dashboards |
 | `authentik.snarfnet.net` | HTTPS | `140.44.4.29` (K3s VIP) | 444 | Authentik SSO Portal |

scripts/check_ftl_db.py

@@ -19,3 +19,4 @@ out, err = exec_cmd('k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-F
 print("Integrity:", out, "Err:", err)
 
 client.close()
+

scripts/check_ftl_log_spaceport.py

@@ -0,0 +1,10 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.41", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- grep -A 10 "Database version is 22" /var/log/pihole/FTL.log'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Spaceport:\n", stdout.read().decode())
+client.close()

scripts/check_ftl_startup.py

@@ -0,0 +1,11 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- grep -B 20 "CRIT: Cannot initialize in-memory database" /var/log/pihole/FTL.log'
+stdin, stdout, stderr = client.exec_command(cmd)
+print(stdout.read().decode())
+client.close()
+

scripts/check_ftl_version.py

@@ -0,0 +1,11 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db "SELECT * FROM ftl;"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print(stdout.read().decode())
+client.close()
+

scripts/check_ftl_version_spaceport.py

@@ -0,0 +1,11 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.41", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db "SELECT * FROM ftl;"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Spaceport:", stdout.read().decode())
+client.close()
+

scripts/check_network_table.py

@@ -0,0 +1,13 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+for tbl in ["network", "network_addresses"]:
+    cmd = f'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db "SELECT count(*) FROM {tbl};"'
+    stdin, stdout, stderr = client.exec_command(cmd)
+    print(f"{tbl} count:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+client.close()
+

scripts/check_npm_ai_hosts.py

@@ -0,0 +1,11 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.71", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'python3 -c "import sqlite3; conn=sqlite3.connect(\'/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite\'); print(conn.execute(\'SELECT id, domain_names, forward_host, forward_port, enabled FROM proxy_host;\').fetchall())"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print(stdout.read().decode())
+print(stderr.read().decode())
+client.close()

scripts/check_technitium_shipyardcontrol.py

@@ -0,0 +1,14 @@
+import requests
+import paramiko
+
+# Let's inspect the technitium zone via SSH or API
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/technitium-dns -c technitium-dns -- find /etc/dns/zones -type f -exec grep -H "shipyardcontrol" {} + || true'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Zone file check:", stdout.read().decode())
+
+client.close()
+

scripts/fix_ftl_db_version.py

@@ -0,0 +1,28 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+# Check current version
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db "SELECT * FROM ftl WHERE id=0;"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Before:", stdout.read().decode().strip())
+
+# Update version to 21
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db "UPDATE ftl SET value=21 WHERE id=0;"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Updated:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+# Verify
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db "SELECT * FROM ftl WHERE id=0;"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("After:", stdout.read().decode().strip())
+
+# Restart FTL or reload
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole restartdns'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Restart output:", stdout.read().decode().strip())
+
+client.close()
+

scripts/remove_shipyardcontrol_npm.py

@@ -0,0 +1,23 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.71", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+# Disable or delete proxy host 29 in NPM database
+cmd = 'python3 -c "import sqlite3; conn=sqlite3.connect(\'/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite\'); conn.execute(\'DELETE FROM proxy_host WHERE id=29;\'); conn.commit(); print(\'Deleted proxy_host 29\')"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("NPM DB update:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+# Remove generated nginx config file if present
+cmd = 'rm -f /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/29.conf'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Removed 29.conf:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+# Reload nginx across the containers
+cmd = 'docker exec $(docker ps -q -f name=nginx-proxy-manager_app) nginx -s reload'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Nginx reload:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+client.close()
+

scripts/test_pihole_user.py

@@ -0,0 +1,12 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- su -s /bin/sh pihole -c "pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db \'SELECT count(*) FROM query_storage;\'"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Output:", stdout.read().decode())
+print("Error:", stderr.read().decode())
+client.close()
+

scripts/test_storage_isolation.py

@@ -0,0 +1,36 @@
+import paramiko
+import socket
+
+SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def run_ssh_cmd(host, user, cmd, timeout=5):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(host, username=user, key_filename=SSH_KEY, timeout=timeout)
+        stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
+        out = stdout.read().decode('utf-8', errors='replace')
+        err = stderr.read().decode('utf-8', errors='replace')
+        client.close()
+        return True, out, err
+    except Exception as e:
+        return False, "", str(e)
+
+print("=== 1. Testing Storage Drop from Smart Home (14.4.4.44 -> 14.10.10.71:5001) ===")
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "nc -z -w 3 14.10.10.71 5001; echo ExitCode=$?")
+print(f"Smart Home to NAS 1: {out.strip() or err.strip()}")
+
+print("\n=== 2. Testing Storage Drop from Smart Home (14.4.4.44 -> 14.10.10.72:5001) ===")
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "nc -z -w 3 14.10.10.72 5001; echo ExitCode=$?")
+print(f"Smart Home to NAS 2: {out.strip() or err.strip()}")
+
+print("\n=== 3. Testing Storage Access from LAN (Workstation 14.44.4.108 -> 14.10.10.71:5001) ===")
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s.settimeout(3)
+try:
+    s.connect(("14.10.10.71", 5001))
+    print("Workstation to NAS 1: SUCCESS (Port 5001 open)")
+    s.close()
+except Exception as e:
+    print(f"Workstation to NAS 1: FAILED ({e})")
+