feat: add infrastructure diagnostic and monitoring scripts

scripts/check_all_npm_hosts.py

@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+py_query = """
+import sqlite3
+conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
+cur = conn.cursor()
+cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host')
+for r in cur.fetchall():
+    print(r)
+conn.close()
+"""
+
+stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"')
+print("All NPM hosts:\n", stdout.read().decode())
+
+c.close()

scripts/check_authentik_events.py

@@ -0,0 +1,35 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+AK_BASE = "https://authentik.snarfnet.net"
+AK_TOKEN = "SnarfNetPermanentAutomationToken2026"
+
+headers = {
+    "Authorization": f"Bearer {AK_TOKEN}",
+    "Content-Type": "application/json",
+    "Accept": "application/json"
+}
+
+# Fetch recent events
+req = urllib.request.Request(f"{AK_BASE}/api/v3/events/events/?page_size=20", headers=headers)
+try:
+    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+        data = json.loads(r.read().decode())
+        for ev in data.get("results", []):
+            action = ev.get("action")
+            user = ev.get("user", {}).get("username")
+            context = ev.get("context", {})
+            created = ev.get("created")
+            print(f"[{created}] Action: {action} | User: {user}")
+            if "message" in context or "redirect_uri" in str(context) or "error" in str(context) or "client_id" in str(context):
+                print(f"   Context: {json.dumps(context, indent=2)}")
+except Exception as e:
+    print(f"Error querying Authentik: {e}")

scripts/check_dns_pods_live.py

@@ -14,3 +14,4 @@ def check_cluster(ip, name):
 
 check_cluster("140.44.4.11", "Aerodrome")
 check_cluster("140.44.4.41", "Spaceport")
+

scripts/check_dns_resolutions.py

@@ -19,3 +19,4 @@ print("--- DNS check via ns2 (140.44.4.65) ---")
 print(run("dig @140.44.4.65 +short pihole1.snarfnet.net pihole2.snarfnet.net dns1.snarfnet.net dns2.snarfnet.net"))
 
 c.close()
+

scripts/check_ha_npm.py

@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+py_query = """
+import sqlite3
+conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
+cur = conn.cursor()
+cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled, advanced_config FROM proxy_host WHERE domain_names LIKE "%ha%" OR domain_names LIKE "%home%" OR forward_host LIKE "%14.4.4.20%"')
+rows = cur.fetchall()
+for r in rows:
+    print(r)
+conn.close()
+"""
+
+stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"')
+print("Home Assistant in NPM:\n", stdout.read().decode())
+
+c.close()

scripts/check_node_vlans.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+for ip, name in [("140.44.4.71", "Swarm/NPM"), ("140.44.4.11", "Aerodrome ctl01"), ("140.44.4.21", "Aerodrome wrk01")]:
+    print(f"\n=== {name} ({ip}) interfaces ===")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(ip, username="root", key_filename=key_path, timeout=5)
+    stdin, stdout, stderr = c.exec_command("ip -br addr; echo 'VLANs:'; ip -d link show type vlan")
+    print(stdout.read().decode())
+    c.close()
+

scripts/check_npm_docker.py

@@ -14,3 +14,4 @@ stdin, stdout, stderr = ssh.exec_command('docker exec -i $(docker ps -q -f name=
 print("NPM find:\n", stdout.read().decode('utf-8', errors='ignore'))
 
 ssh.close()
+

scripts/check_npm_gotify.py

@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+py_query = """
+import sqlite3
+conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
+cur = conn.cursor()
+cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE domain_names LIKE "%gotify%"')
+print(cur.fetchall())
+conn.close()
+"""
+
+stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"')
+print("Gotify in NPM:\n", stdout.read().decode())
+
+c.close()
+

scripts/check_npm_pihole.py

@@ -27,3 +27,4 @@ err = stderr.read().decode()
 if err: print("Stderr:", err)
 
 c.close()
+

scripts/check_npm_sni.py

@@ -25,3 +25,4 @@ print("--- Curl with --resolve dns2 ---")
 print(run("curl -k -I --resolve dns2.snarfnet.net:443:140.44.4.70 https://dns2.snarfnet.net/"))
 
 c.close()
+

scripts/check_nutserver_net.py

@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(cmd)
+    return stdout.read().decode('utf-8', errors='ignore').strip()
+
+print("iptables check:", run("iptables --version; which iptables nft ipset"))
+print("ip forwarding:", run("cat /proc/sys/net/ipv4/ip_forward"))
+print("canary test ping to gateway:", run("ping -c 2 14.4.4.1"))
+
+c.close()
+

scripts/check_nutserver_resources.py

@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(cmd)
+    return stdout.read().decode('utf-8', errors='ignore').strip()
+
+print("Meminfo:\n", run("free -m"))
+print("CPU info:\n", run("lscpu | grep 'Model name\|Architecture\|CPU(s)'"))
+print("Python:\n", run("python3 --version; pip3 --version 2>/dev/null || echo 'pip3 missing'"))
+
+c.close()
+

scripts/check_nutserver_to_swarm.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c.exec_command("curl -s -I http://140.44.4.71:8090/ | head -n 5")
+print(stdout.read().decode())
+
+c.close()
+

scripts/check_pihole_connectivity.py

@@ -31,3 +31,4 @@ print("\n--- Curl test from NPM node to local VIP (140.44.4.70) for pihole2 ---"
 print(run("curl -k -I -H 'Host: pihole2.snarfnet.net' https://140.44.4.70/"))
 
 c.close()
+

scripts/check_pihole_npm.py

@@ -20,3 +20,4 @@ out = stdout.read().decode('utf-8', errors='ignore')
 print("\nProxy hosts in SQLite:\n", out)
 
 ssh.close()
+

scripts/check_records_dns2.py

@@ -19,3 +19,4 @@ def do_req(ip, host_header, path):
 token = do_req("140.44.4.62", "dns2.snarfnet.net", "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true")['token']
 zone_data = do_req("140.44.4.62", "dns2.snarfnet.net", f"/api/zones/records/get?token={token}&zone=snarfnet.net&domain=pihole1.snarfnet.net")
 print("pihole1 records on dns2:\n", zone_data)
+

scripts/check_swarm_services.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c.exec_command("docker service ls; echo '--- ps ---'; docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'")
+print(stdout.read().decode())
+
+c.close()
+

scripts/flush_pihole_cache.py

@@ -5,14 +5,28 @@ sys.stdout.reconfigure(encoding='utf-8')
 key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
 
 def flush_pihole(ip, name):
+    print(f"\n==========================================")
     print(f"Flushing Pi-hole DNS cache on {name} ({ip})...")
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    ssh.connect(ip, username="root", key_filename=key_path, timeout=5)
+    ssh.connect(ip, username="root", key_filename=key_path, timeout=10)
+    
     stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -l app=pihole -o jsonpath='{.items[0].metadata.name}'")
     pod = stdout.read().decode().strip()
-    stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pihole restartdns")
+    print(f"Pi-hole pod on {name}: {pod}")
-    print(stdout.read().decode().strip())
+    
+    if pod:
+        # Pi-hole v6 uses 'pihole reloaddns' or sending SIGHUP to pihole-FTL
+        stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pihole reloaddns")
+        print("reloaddns output:", stdout.read().decode().strip(), stderr.read().decode().strip())
+        
+        stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pkill -HUP -f pihole-FTL")
+        stdout.channel.recv_exit_status()
+        
+        # Test query from pod directly
+        stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- dig @127.0.0.1 +short pihole1.snarfnet.net pihole2.snarfnet.net dns1.snarfnet.net dns2.snarfnet.net")
+        print("Resolved via 127.0.0.1 on pod:\n" + stdout.read().decode().strip())
+        
     ssh.close()
 
 flush_pihole("140.44.4.11", "Aerodrome")

scripts/inspect_ha_authentik.py

@@ -0,0 +1,35 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+AK_BASE = "https://authentik.snarfnet.net"
+AK_TOKEN = "SnarfNetPermanentAutomationToken2026"
+
+headers = {
+    "Authorization": f"Bearer {AK_TOKEN}",
+    "Content-Type": "application/json",
+    "Accept": "application/json"
+}
+
+req = urllib.request.Request(f"{AK_BASE}/api/v3/providers/oauth2/", headers=headers)
+with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+    data = json.loads(r.read().decode())
+    for p in data.get("results", []):
+        if "home" in p.get("name", "").lower() or "assistant" in p.get("name", "").lower():
+            print("Home Assistant Provider:")
+            print(json.dumps(p, indent=2))
+
+req2 = urllib.request.Request(f"{AK_BASE}/api/v3/providers/proxy/", headers=headers)
+with urllib.request.urlopen(req2, timeout=10, context=ctx) as r:
+    data = json.loads(r.read().decode())
+    for p in data.get("results", []):
+        if "home" in p.get("name", "").lower() or "snarfhome" in str(p).lower():
+            print("Proxy Provider:")
+            print(json.dumps(p, indent=2))

scripts/inspect_nutserver_host.py

@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(cmd)
+    return stdout.read().decode('utf-8', errors='ignore').strip()
+
+print("Hostname:", run("hostname"))
+print("Uname:", run("uname -a"))
+print("OS:", run("cat /etc/os-release | grep PRETTY_NAME"))
+print("IPs:\n", run("ip addr"))
+print("Docker / Podman / Python:\n", run("which docker podman python3 opencanary 2>/dev/null"))
+
+c.close()
+

scripts/inspect_technitium_records.py

@@ -0,0 +1,44 @@
+import urllib.request
+import urllib.parse
+import ssl
+import json
+import http.client
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def do_req(ip, host_header, path):
+    conn = http.client.HTTPSConnection(ip, 443, context=ctx, timeout=8)
+    conn.request("GET", path, headers={"Host": host_header})
+    resp = conn.getresponse()
+    data = resp.read().decode('utf-8')
+    conn.close()
+    return json.loads(data)
+
+def get_token(ip, host_header):
+    path = "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true"
+    data = do_req(ip, host_header, path)
+    return data['token']
+
+for ip, host in [("140.44.4.34", "dns1.snarfnet.net"), ("140.44.4.62", "dns2.snarfnet.net")]:
+    print(f"\n======================================")
+    print(f"Checking {host} ({ip})...")
+    try:
+        token = get_token(ip, host)
+        # Get records in snarfnet.net
+        p = urllib.parse.urlencode({"token": token, "domain": "snarfnet.net", "zone": "snarfnet.net", "listZone": "true"})
+        res = do_req(ip, host, f"/api/zones/records/get?{p}")
+        records = res.get("response", {}).get("records", [])
+        for r in records:
+            name = r.get("name")
+            rtype = r.get("type")
+            rdata = r.get("rData", {})
+            if any(k in name for k in ["pihole", "dns", "ns"]):
+                print(f"  {name} ({rtype}) -> {rdata}")
+    except Exception as e:
+        print(f"  Error: {e}")
+

scripts/test_dns_record_ips.py

@@ -28,3 +28,4 @@ for server in ["140.44.4.36", "140.44.4.65"]:
     for dom in ["pihole1.snarfnet.net", "pihole2.snarfnet.net", "dns1.snarfnet.net", "dns2.snarfnet.net", "ns1.snarfnet.net", "ns2.snarfnet.net"]:
         ip = query_ip(server, dom)
         print(f"[{server}] {dom:<25} -> {ip}")
+