feat: add diagnostic and management scripts for DNS, NPM, and cluster infrastructure

This commit is contained in:
2026-11-06 19:08:19 -05:00
parent 8e1c5f1e66
commit d5c2f6eae5
19 changed files with 559 additions and 8 deletions

View File

@@ -0,0 +1,16 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
def check_cluster(ip, name):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -o wide")
print(f"Pods on {name} ({ip}):\n", stdout.read().decode())
ssh.close()
check_cluster("140.44.4.11", "Aerodrome")
check_cluster("140.44.4.41", "Spaceport")

View File

@@ -0,0 +1,21 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(cmd)
return stdout.read().decode('utf-8', errors='ignore') + stderr.read().decode('utf-8', errors='ignore')
print("--- DNS check via ns1 (140.44.4.36) ---")
print(run("dig @140.44.4.36 +short pihole1.snarfnet.net pihole2.snarfnet.net dns1.snarfnet.net dns2.snarfnet.net"))
print("--- DNS check via ns2 (140.44.4.65) ---")
print(run("dig @140.44.4.65 +short pihole1.snarfnet.net pihole2.snarfnet.net dns1.snarfnet.net dns2.snarfnet.net"))
c.close()

View File

@@ -0,0 +1,18 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = ssh.exec_command("ip -br a")
print("IP Aerodrome (140.44.4.11):\n", stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command("k3s kubectl get nodes -o wide")
print("Aerodrome nodes:\n", stdout.read().decode())
ssh.close()

View File

@@ -0,0 +1,16 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
stdin, stdout, stderr = ssh.exec_command('docker ps')
print("Docker ps:\n", stdout.read().decode('utf-8', errors='ignore'))
stdin, stdout, stderr = ssh.exec_command('docker exec -i $(docker ps -q -f name=app -f name=npm) find /data -name "*.sqlite* -o -name "*.conf" 2>/dev/null')
print("NPM find:\n", stdout.read().decode('utf-8', errors='ignore'))
ssh.close()

View File

@@ -0,0 +1,29 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
py_query = """
import sqlite3
conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
cur = conn.cursor()
cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host')
rows = cur.fetchall()
for r in rows:
domains = r[1]
if any(k in domains for k in ['pihole', 'dns', 'ns']):
print(r)
conn.close()
"""
stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"')
print("SQLite Proxy Hosts:\n", stdout.read().decode())
err = stderr.read().decode()
if err: print("Stderr:", err)
c.close()

27
scripts/check_npm_sni.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(cmd)
return stdout.read().decode('utf-8', errors='ignore') + stderr.read().decode('utf-8', errors='ignore')
print("--- Curl with --resolve pihole1 ---")
print(run("curl -k -I --resolve pihole1.snarfnet.net:443:140.44.4.70 https://pihole1.snarfnet.net/"))
print("--- Curl with --resolve pihole2 ---")
print(run("curl -k -I --resolve pihole2.snarfnet.net:443:140.44.4.70 https://pihole2.snarfnet.net/"))
print("--- Curl with --resolve dns1 ---")
print(run("curl -k -I --resolve dns1.snarfnet.net:443:140.44.4.70 https://dns1.snarfnet.net/"))
print("--- Curl with --resolve dns2 ---")
print(run("curl -k -I --resolve dns2.snarfnet.net:443:140.44.4.70 https://dns2.snarfnet.net/"))
c.close()

View File

@@ -0,0 +1,33 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(cmd)
return stdout.read().decode('utf-8', errors='ignore') + stderr.read().decode('utf-8', errors='ignore')
print("--- 55.conf (pihole1) ---")
print(run("cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/55.conf"))
print("\n--- 43.conf (pihole2) ---")
print(run("cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/43.conf"))
print("\n--- Curl test from NPM node to Aerodrome Traefik (140.44.4.34) for pihole1 ---")
print(run("curl -k -I -H 'Host: pihole1.snarfnet.net' https://140.44.4.34/"))
print("\n--- Curl test from NPM node to Spaceport Traefik (140.44.4.62) for pihole2 ---")
print(run("curl -k -I -H 'Host: pihole2.snarfnet.net' https://140.44.4.62/"))
print("\n--- Curl test from NPM node to local VIP (140.44.4.70) for pihole1 ---")
print(run("curl -k -I -H 'Host: pihole1.snarfnet.net' https://140.44.4.70/"))
print("\n--- Curl test from NPM node to local VIP (140.44.4.70) for pihole2 ---")
print(run("curl -k -I -H 'Host: pihole2.snarfnet.net' https://140.44.4.70/"))
c.close()

View File

@@ -0,0 +1,22 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
except Exception:
ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
stdin, stdout, stderr = ssh.exec_command('grep -r -i "pihole" /data/nginx/proxy_host/ 2>/dev/null')
out = stdout.read().decode('utf-8', errors='ignore')
print("Grep pihole in proxy_host:\n", out)
# Also let's inspect sqlite database in NPM
stdin, stdout, stderr = ssh.exec_command('sqlite3 /data/database.sqlite "SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE domain_names LIKE \'%pihole%\' OR domain_names LIKE \'%dns%\';"')
out = stdout.read().decode('utf-8', errors='ignore')
print("\nProxy hosts in SQLite:\n", out)
ssh.close()

View File

@@ -0,0 +1,21 @@
import urllib.request
import urllib.parse
import ssl
import json
import http.client
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def do_req(ip, host_header, path):
conn = http.client.HTTPSConnection(ip, 443, context=ctx, timeout=8)
conn.request("GET", path, headers={"Host": host_header})
resp = conn.getresponse()
data = resp.read().decode('utf-8')
conn.close()
return json.loads(data)
token = do_req("140.44.4.62", "dns2.snarfnet.net", "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true")['token']
zone_data = do_req("140.44.4.62", "dns2.snarfnet.net", f"/api/zones/records/get?token={token}&zone=snarfnet.net&domain=pihole1.snarfnet.net")
print("pihole1 records on dns2:\n", zone_data)

View File

@@ -2,13 +2,13 @@ import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl get nodes -o wide")
print("=== SPACEPORT NODES ===")
print(stdout.read().decode('utf-8'))
client.close()
stdin, stdout, stderr = ssh.exec_command("k3s kubectl get nodes -o wide")
print("Spaceport nodes:\n", stdout.read().decode())
ssh.close()

View File

@@ -0,0 +1,73 @@
import urllib.request
import urllib.parse
import ssl
import json
import http.client
import sys
sys.stdout.reconfigure(encoding='utf-8')
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def do_req(ip, host_header, path):
conn = http.client.HTTPSConnection(ip, 443, context=ctx, timeout=8)
conn.request("GET", path, headers={"Host": host_header})
resp = conn.getresponse()
data = resp.read().decode('utf-8')
conn.close()
return json.loads(data)
def get_token(ip, host_header):
path = "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true"
data = do_req(ip, host_header, path)
return data['token']
def update_server(ip, host_header):
print(f"\n=======================================================")
print(f"Connecting to {host_header} via {ip}...")
token = get_token(ip, host_header)
print(f"Token acquired for {host_header}")
# Records to fix
records = [
("pihole1", "140.44.4.70"),
("pihole2", "140.44.4.70"),
("dns1", "140.44.4.70"),
("dns2", "140.44.4.70"),
("ns1", "140.44.4.36"),
("ns2", "140.44.4.65"),
]
for dom, target_ip in records:
full_dom = f"{dom}.snarfnet.net"
# 1. Delete old record
del_p = urllib.parse.urlencode({"token": token, "domain": full_dom, "zone": "snarfnet.net", "type": "A"})
try:
do_req(ip, host_header, f"/api/zones/records/delete?{del_p}")
except Exception:
pass
# 2. Add correct record
use_ptr = "true" if dom.startswith("ns") else "false"
add_p = urllib.parse.urlencode({
"token": token,
"domain": full_dom,
"zone": "snarfnet.net",
"type": "A",
"ipAddress": target_ip,
"ptr": use_ptr,
"overwrite": "true",
"ttl": 60
})
res = do_req(ip, host_header, f"/api/zones/records/add?{add_p}")
print(f" ✓ {full_dom} -> {target_ip}: {res.get('status')}")
# Update Aerodrome Technitium via Traefik 140.44.4.34
update_server("140.44.4.34", "dns1.snarfnet.net")
# Update Spaceport Technitium via Traefik 140.44.4.62
update_server("140.44.4.62", "dns2.snarfnet.net")
print("\nSuccessfully updated DNS records on both Technitium servers!")

View File

@@ -0,0 +1,19 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
def flush_pihole(ip, name):
print(f"Flushing Pi-hole DNS cache on {name} ({ip})...")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -l app=pihole -o jsonpath='{.items[0].metadata.name}'")
pod = stdout.read().decode().strip()
stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod} -- pihole restartdns")
print(stdout.read().decode().strip())
ssh.close()
flush_pihole("140.44.4.11", "Aerodrome")
flush_pihole("140.44.4.41", "Spaceport")

View File

@@ -0,0 +1,148 @@
import urllib.request
import urllib.parse
import ssl
import json
import sys
sys.stdout.reconfigure(encoding='utf-8')
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
A_RECORDS = [
# Gateways
("gateway", "140.44.4.1"),
("udm", "140.44.4.1"),
("gateway-lan", "14.44.4.1"),
("gateway-mgmt", "140.44.44.1"),
("gateway-storage", "14.10.10.1"),
("gateway-smarthome", "14.4.4.1"),
("gateway-media", "192.168.1.1"),
# VIPs
("npm", "140.44.4.70"),
("npm-vip", "140.44.4.70"),
("*", "140.44.4.70"), # Wildcard for all reverse-proxied services
("aero-traefik", "140.44.4.34"),
("space-traefik", "140.44.4.62"),
("dns1", "140.44.4.36"),
("pihole1", "140.44.4.36"),
("dns2", "140.44.4.65"),
("pihole2", "140.44.4.65"),
# Storage
("sn-nas", "14.10.10.71"),
("snarfnas", "14.10.10.72"),
# Aerodrome Control Plane
("sninfr2lnxctl01", "140.44.4.11"),
("sninfr2lnxctl02", "140.44.4.12"),
("sninfr2lnxctl03", "140.44.4.13"),
# Aerodrome Workers
("sninfr2lnxwrk01", "140.44.4.21"),
("sninfr2lnxwrk02", "140.44.4.22"),
("sninfr2lnxwrk03", "140.44.4.23"),
("sninfr2lnxwrk04", "140.44.4.24"),
("sninfr2lnxwrk05", "140.44.4.25"),
("sninfr2lnxwrk06", "140.44.4.26"),
("sninfr2lnxwrk07", "140.44.4.27"),
# Spaceport Control Plane
("sninfr0lnxctl01", "140.44.4.41"),
("sninfr0lnxctl02", "140.44.4.42"),
("sninfr0lnxctl03", "140.44.4.43"),
("sninfr0lnxctl04", "140.44.4.44"),
# Spaceport Workers
("sninfr0lnxwrk01", "140.44.4.51"),
("sninfr0lnxwrk02", "140.44.4.52"),
("sninfr0lnxwrk03", "140.44.4.53"),
("sninfr0lnxwrk04", "140.44.4.54"),
("sninfr0lnxwrk05", "140.44.4.55"),
("sninfr0lnxwrk06", "140.44.4.56"),
# Swarm Node
("sninfr1lnxctl01", "140.44.4.71"),
# Harvester
("spacedock", "140.44.4.90"),
]
PTR_ZONES = [
"4.44.140.in-addr.arpa",
"44.44.140.in-addr.arpa",
"10.10.14.in-addr.arpa",
"4.44.14.in-addr.arpa",
"4.4.14.in-addr.arpa",
"1.168.192.in-addr.arpa",
]
def get_token(host):
url = f"https://{host}/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true"
with urllib.request.urlopen(urllib.request.Request(url), context=ctx, timeout=8) as resp:
return json.loads(resp.read().decode())['token']
def provision_server(host):
print(f"\n=======================================================")
print(f"Provisioning Technitium on {host}...")
token = get_token(host)
print(f"Token acquired for {host}")
# 1. Create Forward Zone snarfnet.net
create_zone_url = f"https://{host}/api/zones/create?token={token}&zone=snarfnet.net&type=Primary"
try:
with urllib.request.urlopen(urllib.request.Request(create_zone_url), context=ctx, timeout=8) as resp:
print(f" ✓ Created forward zone snarfnet.net on {host}")
except urllib.error.HTTPError as e:
err = e.read().decode()
if "already exists" in err:
print(f" - Forward zone snarfnet.net already exists on {host}")
else:
print(f" ✗ Error creating snarfnet.net: {err}")
# 2. Create Reverse PTR Zones
for z in PTR_ZONES:
cz_url = f"https://{host}/api/zones/create?token={token}&zone={z}&type=Primary"
try:
with urllib.request.urlopen(urllib.request.Request(cz_url), context=ctx, timeout=8) as resp:
print(f" ✓ Created reverse zone {z} on {host}")
except urllib.error.HTTPError as e:
err = e.read().decode()
if "already exists" in err:
print(f" - Reverse zone {z} already exists on {host}")
else:
print(f" ✗ Error creating reverse zone {z}: {err}")
# 3. Add A records + Auto PTR
print(f"\nAdding {len(A_RECORDS)} authoritative records on {host}...")
added = 0
for name, ip in A_RECORDS:
domain = f"{name}.snarfnet.net" if name != "@" else "snarfnet.net"
# Don't create PTR for wildcard
use_ptr = "false" if name == "*" else "true"
params = {
"token": token,
"domain": domain,
"zone": "snarfnet.net",
"type": "A",
"ipAddress": ip,
"ptr": use_ptr,
"overwrite": "true",
"ttl": 3600
}
add_url = f"https://{host}/api/zones/records/add?{urllib.parse.urlencode(params)}"
try:
with urllib.request.urlopen(urllib.request.Request(add_url), context=ctx, timeout=8) as resp:
added += 1
except Exception as e:
print(f" ✗ Failed adding {domain} -> {ip}: {e}")
print(f" ✓ Successfully added/updated {added} A and PTR records on {host}")
provision_server("dns1.snarfnet.net")
provision_server("dns2.snarfnet.net")
print("\nDone provisioning both Technitium DNS servers!")

View File

@@ -0,0 +1,30 @@
import socket
import sys
def make_query(name):
import random
txid = random.randint(1, 65535)
header = txid.to_bytes(2, 'big') + (0x0100).to_bytes(2, 'big') + (1).to_bytes(2, 'big') + b'\x00\x00\x00\x00\x00\x00'
qname = b''.join([bytes([len(p)]) + p.encode('ascii') for p in name.split('.')]) + b'\x00'
return header + qname + (1).to_bytes(2, 'big') + (1).to_bytes(2, 'big')
def query_ip(server, domain):
packet = make_query(domain)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(2.0)
try:
s.sendto(packet, (server, 53))
data, _ = s.recvfrom(1024)
# Parse answer IP (last 4 bytes of first A record)
if len(data) >= 16:
ip = f"{data[-4]}.{data[-3]}.{data[-2]}.{data[-1]}"
return ip
except Exception as e:
return str(e)
finally:
s.close()
for server in ["140.44.4.36", "140.44.4.65"]:
for dom in ["pihole1.snarfnet.net", "pihole2.snarfnet.net", "dns1.snarfnet.net", "dns2.snarfnet.net", "ns1.snarfnet.net", "ns2.snarfnet.net"]:
ip = query_ip(server, dom)
print(f"[{server}] {dom:<25} -> {ip}")

62
scripts/test_dns_vips.py Normal file
View File

@@ -0,0 +1,62 @@
import socket
import sys
sys.stdout.reconfigure(encoding='utf-8')
# Simple DNS query packet generator
def make_query(name, qtype=1): # 1=A, 12=PTR
import random
txid = random.randint(1, 65535)
flags = 0x0100 # standard query, recursion desired
qdcount = 1
header = txid.to_bytes(2, 'big') + flags.to_bytes(2, 'big') + qdcount.to_bytes(2, 'big') + b'\x00\x00\x00\x00\x00\x00'
qname = b''.join([bytes([len(p)]) + p.encode('ascii') for p in name.split('.')]) + b'\x00'
question = qname + qtype.to_bytes(2, 'big') + (1).to_bytes(2, 'big') # class IN
return header + question, txid
def query_server(server, name, qtype=1, port=53):
packet, txid = make_query(name, qtype)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(3.0)
try:
s.sendto(packet, (server, port))
data, _ = s.recvfrom(1024)
# Parse basic response
ancount = int.from_bytes(data[6:8], 'big')
return True, ancount
except Exception as e:
return False, str(e)
finally:
s.close()
targets = [
("Aerodrome DNS VIP (140.44.4.36)", "140.44.4.36"),
("Spaceport DNS VIP (140.44.4.65)", "140.44.4.65"),
]
queries = [
("sninfr2lnxctl01.snarfnet.net", 1),
("sn-nas.snarfnet.net", 1),
("gateway.snarfnet.net", 1),
("app-test.snarfnet.net", 1), # Wildcard
("11.4.44.140.in-addr.arpa", 12), # Reverse PTR
("google.com", 1), # Upstream DoH recursion
]
print("=======================================================")
print(" TESTING DNS VIPS PORT 53 RESOLUTION ")
print("=======================================================")
for s_label, server in targets:
print(f"\n--- Testing {s_label} ---")
for q_name, qtype in queries:
ok, res = query_server(server, q_name, qtype)
if ok and res > 0:
print(f" ✓ {q_name:<30} -> Resolved ({res} answers)")
elif ok and res == 0:
print(f" ? {q_name:<30} -> 0 answers returned")
else:
print(f" ✗ {q_name:<30} -> Error: {res}")
print("\n=======================================================")

13
scripts/test_gotify.py Normal file
View File

@@ -0,0 +1,13 @@
import urllib.request
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
try:
with urllib.request.urlopen("https://gotify.snarfnet.net/version", context=ctx, timeout=5) as resp:
print("Gotify version:", resp.read().decode().strip())
except Exception as e:
print("Gotify error:", e)

View File

@@ -33,3 +33,4 @@ try:
print("Add record response:", json.loads(resp.read().decode()))
except urllib.error.HTTPError as e:
print("HTTPError:", e.code, e.read().decode())

View File

@@ -25,3 +25,4 @@ try:
print("Create zone response:", json.loads(resp.read().decode()))
except urllib.error.HTTPError as e:
print("HTTPError:", e.code, e.read().decode())

View File

@@ -24,3 +24,4 @@ params = {
url = f"https://dns1.snarfnet.net/api/zones/records/delete?{urllib.parse.urlencode(params)}"
with urllib.request.urlopen(urllib.request.Request(url), context=ctx, timeout=8) as resp:
print("Delete record response:", json.loads(resp.read().decode()))