From 0ed293b6ea4c44855ed2d310d84551046210cb41 Mon Sep 17 00:00:00 2001 From: Snarf Date: Fri, 6 Nov 2026 23:12:35 -0500 Subject: [PATCH] feat: add infrastructure diagnostic and monitoring scripts --- scripts/check_all_npm_hosts.py | 24 +++++++++++++++ scripts/check_authentik_events.py | 35 +++++++++++++++++++++ scripts/check_dns_pods_live.py | 1 + scripts/check_dns_resolutions.py | 1 + scripts/check_ha_npm.py | 25 +++++++++++++++ scripts/check_node_vlans.py | 15 +++++++++ scripts/check_npm_docker.py | 1 + scripts/check_npm_gotify.py | 24 +++++++++++++++ scripts/check_npm_pihole.py | 1 + scripts/check_npm_sni.py | 1 + scripts/check_nutserver_net.py | 20 ++++++++++++ scripts/check_nutserver_resources.py | 20 ++++++++++++ scripts/check_nutserver_to_swarm.py | 15 +++++++++ scripts/check_pihole_connectivity.py | 1 + scripts/check_pihole_npm.py | 1 + scripts/check_records_dns2.py | 1 + scripts/check_swarm_services.py | 15 +++++++++ scripts/flush_pihole_cache.py | 20 ++++++++++-- scripts/inspect_ha_authentik.py | 35 +++++++++++++++++++++ scripts/inspect_nutserver_host.py | 22 ++++++++++++++ scripts/inspect_technitium_records.py | 44 +++++++++++++++++++++++++++ scripts/test_dns_record_ips.py | 1 + 22 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 scripts/check_all_npm_hosts.py create mode 100644 scripts/check_authentik_events.py create mode 100644 scripts/check_ha_npm.py create mode 100644 scripts/check_node_vlans.py create mode 100644 scripts/check_npm_gotify.py create mode 100644 scripts/check_nutserver_net.py create mode 100644 scripts/check_nutserver_resources.py create mode 100644 scripts/check_nutserver_to_swarm.py create mode 100644 scripts/check_swarm_services.py create mode 100644 scripts/inspect_ha_authentik.py create mode 100644 scripts/inspect_nutserver_host.py create mode 100644 scripts/inspect_technitium_records.py diff --git a/scripts/check_all_npm_hosts.py b/scripts/check_all_npm_hosts.py new file mode 100644 index 0000000..a5406ea --- /dev/null +++ b/scripts/check_all_npm_hosts.py @@ -0,0 +1,24 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10) + +py_query = """ +import sqlite3 +conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite') +cur = conn.cursor() +cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host') +for r in cur.fetchall(): + print(r) +conn.close() +""" + +stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"') +print("All NPM hosts:\n", stdout.read().decode()) + +c.close() diff --git a/scripts/check_authentik_events.py b/scripts/check_authentik_events.py new file mode 100644 index 0000000..d39d30a --- /dev/null +++ b/scripts/check_authentik_events.py @@ -0,0 +1,35 @@ +import urllib.request +import ssl +import json +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +AK_BASE = "https://authentik.snarfnet.net" +AK_TOKEN = "SnarfNetPermanentAutomationToken2026" + +headers = { + "Authorization": f"Bearer {AK_TOKEN}", + "Content-Type": "application/json", + "Accept": "application/json" +} + +# Fetch recent events +req = urllib.request.Request(f"{AK_BASE}/api/v3/events/events/?page_size=20", headers=headers) +try: + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + data = json.loads(r.read().decode()) + for ev in data.get("results", []): + action = ev.get("action") + user = ev.get("user", {}).get("username") + context = ev.get("context", {}) + created = ev.get("created") + print(f"[{created}] Action: {action} | User: {user}") + if "message" in context or "redirect_uri" in str(context) or "error" in str(context) or "client_id" in str(context): + print(f" Context: {json.dumps(context, indent=2)}") +except Exception as e: + print(f"Error querying Authentik: {e}") diff --git a/scripts/check_dns_pods_live.py b/scripts/check_dns_pods_live.py index 94205f2..893f20c 100644 --- a/scripts/check_dns_pods_live.py +++ b/scripts/check_dns_pods_live.py @@ -14,3 +14,4 @@ def check_cluster(ip, name): check_cluster("140.44.4.11", "Aerodrome") check_cluster("140.44.4.41", "Spaceport") + diff --git a/scripts/check_dns_resolutions.py b/scripts/check_dns_resolutions.py index f70026c..64726ca 100644 --- a/scripts/check_dns_resolutions.py +++ b/scripts/check_dns_resolutions.py @@ -19,3 +19,4 @@ print("--- DNS check via ns2 (140.44.4.65) ---") print(run("dig @140.44.4.65 +short pihole1.snarfnet.net pihole2.snarfnet.net dns1.snarfnet.net dns2.snarfnet.net")) c.close() + diff --git a/scripts/check_ha_npm.py b/scripts/check_ha_npm.py new file mode 100644 index 0000000..8406018 --- /dev/null +++ b/scripts/check_ha_npm.py @@ -0,0 +1,25 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10) + +py_query = """ +import sqlite3 +conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite') +cur = conn.cursor() +cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled, advanced_config FROM proxy_host WHERE domain_names LIKE "%ha%" OR domain_names LIKE "%home%" OR forward_host LIKE "%14.4.4.20%"') +rows = cur.fetchall() +for r in rows: + print(r) +conn.close() +""" + +stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"') +print("Home Assistant in NPM:\n", stdout.read().decode()) + +c.close() diff --git a/scripts/check_node_vlans.py b/scripts/check_node_vlans.py new file mode 100644 index 0000000..3815d9d --- /dev/null +++ b/scripts/check_node_vlans.py @@ -0,0 +1,15 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +for ip, name in [("140.44.4.71", "Swarm/NPM"), ("140.44.4.11", "Aerodrome ctl01"), ("140.44.4.21", "Aerodrome wrk01")]: + print(f"\n=== {name} ({ip}) interfaces ===") + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username="root", key_filename=key_path, timeout=5) + stdin, stdout, stderr = c.exec_command("ip -br addr; echo 'VLANs:'; ip -d link show type vlan") + print(stdout.read().decode()) + c.close() + diff --git a/scripts/check_npm_docker.py b/scripts/check_npm_docker.py index a0bd14a..36d6a94 100644 --- a/scripts/check_npm_docker.py +++ b/scripts/check_npm_docker.py @@ -14,3 +14,4 @@ stdin, stdout, stderr = ssh.exec_command('docker exec -i $(docker ps -q -f name= print("NPM find:\n", stdout.read().decode('utf-8', errors='ignore')) ssh.close() + diff --git a/scripts/check_npm_gotify.py b/scripts/check_npm_gotify.py new file mode 100644 index 0000000..583af54 --- /dev/null +++ b/scripts/check_npm_gotify.py @@ -0,0 +1,24 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10) + +py_query = """ +import sqlite3 +conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite') +cur = conn.cursor() +cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE domain_names LIKE "%gotify%"') +print(cur.fetchall()) +conn.close() +""" + +stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"') +print("Gotify in NPM:\n", stdout.read().decode()) + +c.close() + diff --git a/scripts/check_npm_pihole.py b/scripts/check_npm_pihole.py index 69a1f01..1e9c568 100644 --- a/scripts/check_npm_pihole.py +++ b/scripts/check_npm_pihole.py @@ -27,3 +27,4 @@ err = stderr.read().decode() if err: print("Stderr:", err) c.close() + diff --git a/scripts/check_npm_sni.py b/scripts/check_npm_sni.py index f326115..71087b4 100644 --- a/scripts/check_npm_sni.py +++ b/scripts/check_npm_sni.py @@ -25,3 +25,4 @@ print("--- Curl with --resolve dns2 ---") print(run("curl -k -I --resolve dns2.snarfnet.net:443:140.44.4.70 https://dns2.snarfnet.net/")) c.close() + diff --git a/scripts/check_nutserver_net.py b/scripts/check_nutserver_net.py new file mode 100644 index 0000000..62368d1 --- /dev/null +++ b/scripts/check_nutserver_net.py @@ -0,0 +1,20 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(cmd) + return stdout.read().decode('utf-8', errors='ignore').strip() + +print("iptables check:", run("iptables --version; which iptables nft ipset")) +print("ip forwarding:", run("cat /proc/sys/net/ipv4/ip_forward")) +print("canary test ping to gateway:", run("ping -c 2 14.4.4.1")) + +c.close() + diff --git a/scripts/check_nutserver_resources.py b/scripts/check_nutserver_resources.py new file mode 100644 index 0000000..cb92865 --- /dev/null +++ b/scripts/check_nutserver_resources.py @@ -0,0 +1,20 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(cmd) + return stdout.read().decode('utf-8', errors='ignore').strip() + +print("Meminfo:\n", run("free -m")) +print("CPU info:\n", run("lscpu | grep 'Model name\|Architecture\|CPU(s)'")) +print("Python:\n", run("python3 --version; pip3 --version 2>/dev/null || echo 'pip3 missing'")) + +c.close() + diff --git a/scripts/check_nutserver_to_swarm.py b/scripts/check_nutserver_to_swarm.py new file mode 100644 index 0000000..9606fb5 --- /dev/null +++ b/scripts/check_nutserver_to_swarm.py @@ -0,0 +1,15 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10) + +stdin, stdout, stderr = c.exec_command("curl -s -I http://140.44.4.71:8090/ | head -n 5") +print(stdout.read().decode()) + +c.close() + diff --git a/scripts/check_pihole_connectivity.py b/scripts/check_pihole_connectivity.py index 36266b1..1aa67c3 100644 --- a/scripts/check_pihole_connectivity.py +++ b/scripts/check_pihole_connectivity.py @@ -31,3 +31,4 @@ print("\n--- Curl test from NPM node to local VIP (140.44.4.70) for pihole2 ---" print(run("curl -k -I -H 'Host: pihole2.snarfnet.net' https://140.44.4.70/")) c.close() + diff --git a/scripts/check_pihole_npm.py b/scripts/check_pihole_npm.py index 49509cd..41f155c 100644 --- a/scripts/check_pihole_npm.py +++ b/scripts/check_pihole_npm.py @@ -20,3 +20,4 @@ out = stdout.read().decode('utf-8', errors='ignore') print("\nProxy hosts in SQLite:\n", out) ssh.close() + diff --git a/scripts/check_records_dns2.py b/scripts/check_records_dns2.py index ed89966..a718b84 100644 --- a/scripts/check_records_dns2.py +++ b/scripts/check_records_dns2.py @@ -19,3 +19,4 @@ def do_req(ip, host_header, path): token = do_req("140.44.4.62", "dns2.snarfnet.net", "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true")['token'] zone_data = do_req("140.44.4.62", "dns2.snarfnet.net", f"/api/zones/records/get?token={token}&zone=snarfnet.net&domain=pihole1.snarfnet.net") print("pihole1 records on dns2:\n", zone_data) + diff --git a/scripts/check_swarm_services.py b/scripts/check_swarm_services.py new file mode 100644 index 0000000..f843b1d --- /dev/null +++ b/scripts/check_swarm_services.py @@ -0,0 +1,15 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10) + +stdin, stdout, stderr = c.exec_command("docker service ls; echo '--- ps ---'; docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'") +print(stdout.read().decode()) + +c.close() + diff --git a/scripts/flush_pihole_cache.py b/scripts/flush_pihole_cache.py index cc9586b..e3aeb10 100644 --- a/scripts/flush_pihole_cache.py +++ b/scripts/flush_pihole_cache.py @@ -5,14 +5,28 @@ sys.stdout.reconfigure(encoding='utf-8') key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' def flush_pihole(ip, name): + print(f"\n==========================================") print(f"Flushing Pi-hole DNS cache on {name} ({ip})...") ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, username="root", key_filename=key_path, timeout=5) + ssh.connect(ip, username="root", key_filename=key_path, timeout=10) + stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -l app=pihole -o jsonpath='{.items[0].metadata.name}'") pod = stdout.read().decode().strip() - stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pihole restartdns") - print(stdout.read().decode().strip()) + print(f"Pi-hole pod on {name}: {pod}") + + if pod: + # Pi-hole v6 uses 'pihole reloaddns' or sending SIGHUP to pihole-FTL + stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pihole reloaddns") + print("reloaddns output:", stdout.read().decode().strip(), stderr.read().decode().strip()) + + stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pkill -HUP -f pihole-FTL") + stdout.channel.recv_exit_status() + + # Test query from pod directly + stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- dig @127.0.0.1 +short pihole1.snarfnet.net pihole2.snarfnet.net dns1.snarfnet.net dns2.snarfnet.net") + print("Resolved via 127.0.0.1 on pod:\n" + stdout.read().decode().strip()) + ssh.close() flush_pihole("140.44.4.11", "Aerodrome") diff --git a/scripts/inspect_ha_authentik.py b/scripts/inspect_ha_authentik.py new file mode 100644 index 0000000..0c93eb8 --- /dev/null +++ b/scripts/inspect_ha_authentik.py @@ -0,0 +1,35 @@ +import urllib.request +import ssl +import json +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +AK_BASE = "https://authentik.snarfnet.net" +AK_TOKEN = "SnarfNetPermanentAutomationToken2026" + +headers = { + "Authorization": f"Bearer {AK_TOKEN}", + "Content-Type": "application/json", + "Accept": "application/json" +} + +req = urllib.request.Request(f"{AK_BASE}/api/v3/providers/oauth2/", headers=headers) +with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + data = json.loads(r.read().decode()) + for p in data.get("results", []): + if "home" in p.get("name", "").lower() or "assistant" in p.get("name", "").lower(): + print("Home Assistant Provider:") + print(json.dumps(p, indent=2)) + +req2 = urllib.request.Request(f"{AK_BASE}/api/v3/providers/proxy/", headers=headers) +with urllib.request.urlopen(req2, timeout=10, context=ctx) as r: + data = json.loads(r.read().decode()) + for p in data.get("results", []): + if "home" in p.get("name", "").lower() or "snarfhome" in str(p).lower(): + print("Proxy Provider:") + print(json.dumps(p, indent=2)) diff --git a/scripts/inspect_nutserver_host.py b/scripts/inspect_nutserver_host.py new file mode 100644 index 0000000..3ed368b --- /dev/null +++ b/scripts/inspect_nutserver_host.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(cmd) + return stdout.read().decode('utf-8', errors='ignore').strip() + +print("Hostname:", run("hostname")) +print("Uname:", run("uname -a")) +print("OS:", run("cat /etc/os-release | grep PRETTY_NAME")) +print("IPs:\n", run("ip addr")) +print("Docker / Podman / Python:\n", run("which docker podman python3 opencanary 2>/dev/null")) + +c.close() + diff --git a/scripts/inspect_technitium_records.py b/scripts/inspect_technitium_records.py new file mode 100644 index 0000000..6385815 --- /dev/null +++ b/scripts/inspect_technitium_records.py @@ -0,0 +1,44 @@ +import urllib.request +import urllib.parse +import ssl +import json +import http.client +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +def do_req(ip, host_header, path): + conn = http.client.HTTPSConnection(ip, 443, context=ctx, timeout=8) + conn.request("GET", path, headers={"Host": host_header}) + resp = conn.getresponse() + data = resp.read().decode('utf-8') + conn.close() + return json.loads(data) + +def get_token(ip, host_header): + path = "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true" + data = do_req(ip, host_header, path) + return data['token'] + +for ip, host in [("140.44.4.34", "dns1.snarfnet.net"), ("140.44.4.62", "dns2.snarfnet.net")]: + print(f"\n======================================") + print(f"Checking {host} ({ip})...") + try: + token = get_token(ip, host) + # Get records in snarfnet.net + p = urllib.parse.urlencode({"token": token, "domain": "snarfnet.net", "zone": "snarfnet.net", "listZone": "true"}) + res = do_req(ip, host, f"/api/zones/records/get?{p}") + records = res.get("response", {}).get("records", []) + for r in records: + name = r.get("name") + rtype = r.get("type") + rdata = r.get("rData", {}) + if any(k in name for k in ["pihole", "dns", "ns"]): + print(f" {name} ({rtype}) -> {rdata}") + except Exception as e: + print(f" Error: {e}") + diff --git a/scripts/test_dns_record_ips.py b/scripts/test_dns_record_ips.py index ffd251c..a2ca1e6 100644 --- a/scripts/test_dns_record_ips.py +++ b/scripts/test_dns_record_ips.py @@ -28,3 +28,4 @@ for server in ["140.44.4.36", "140.44.4.65"]: for dom in ["pihole1.snarfnet.net", "pihole2.snarfnet.net", "dns1.snarfnet.net", "dns2.snarfnet.net", "ns1.snarfnet.net", "ns2.snarfnet.net"]: ip = query_ip(server, dom) print(f"[{server}] {dom:<25} -> {ip}") +