feat: add script to apply DNS manifests via SSH and kubectl

scripts/apply_dns_updates.py

@@ -1,38 +1,35 @@
 import paramiko
-import sys
-import time
 
-sys.stdout.reconfigure(encoding='utf-8')
+SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
 
-key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
-
-def apply_to_cluster(ip, name, manifest_path):
-    print(f"Connecting to {name} ({ip})...")
-    with open(manifest_path, "r", encoding="utf-8") as f:
-        content = f.read()
-
-    c = paramiko.SSHClient()
-    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    c.connect(ip, username="root", key_filename=key_path, timeout=10)
+def apply_manifest(host, local_file):
+    with open(local_file, "r") as f:
+        manifest_content = f.read()
     
-    stdin, stdout, stderr = c.exec_command("k3s kubectl apply -f -")
-    stdin.write(content)
-    stdin.channel.shutdown_write()
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(host, username="root", key_filename=SSH_KEY, timeout=10)
     
-    out = stdout.read().decode('utf-8')
-    err = stderr.read().decode('utf-8')
-    print(f"[{name}] Apply Output:\n{out}")
-    if err.strip():
-        print(f"[{name}] Apply Error:\n{err}")
-    c.close()
-
-aero_path = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml"
-space_path = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml"
-
-apply_to_cluster("140.44.4.11", "Aerodrome", aero_path)
-time.sleep(2)
-apply_to_cluster("140.44.4.41", "Spaceport", space_path)
-print("Finished applying manifests to both clusters!")
-
+    # Write to remote /tmp/dns-stack.yaml and kubectl apply
+    sftp = client.open_sftp()
+    with sftp.open("/tmp/dns-stack.yaml", "w") as f:
+        f.write(manifest_content)
+    sftp.close()
+    
+    stdin, stdout, stderr = client.exec_command("k3s kubectl apply -f /tmp/dns-stack.yaml")
+    out = stdout.read().decode()
+    err = stderr.read().decode()
+    client.close()
+    return out, err
 
+print("=== Applying to Aerodrome (140.44.4.11) ===")
+out, err = apply_manifest("140.44.4.11", r"homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml")
+print(out)
+if err:
+    print("Stderr:", err)
 
+print("=== Applying to Spaceport (140.44.4.41) ===")
+out, err = apply_manifest("140.44.4.41", r"homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml")
+print(out)
+if err:
+    print("Stderr:", err)

scripts/check_current_clients.py

@@ -0,0 +1,12 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl exec -n dns deploy/pihole -c pihole -- tail -n 25 /var/log/pihole/pihole.log'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Queries by client on Aerodrome in last 15 min:")
+print(stdout.read().decode())
+print("Stderr:", stderr.read().decode())
+client.close()

scripts/check_pihole_queries.py

@@ -0,0 +1,34 @@
+import sys
+import paramiko
+
+SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def run_ssh_cmd(host, user, cmd, timeout=10):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(host, username=user, key_filename=SSH_KEY, timeout=timeout)
+        stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
+        out = stdout.read().decode('utf-8', errors='replace')
+        err = stderr.read().decode('utf-8', errors='replace')
+        client.close()
+        return True, out, err
+    except Exception as e:
+        return False, "", str(e)
+
+print("=== 1. Aerodrome Pi-hole Query Flow ===")
+ok, out, err = run_ssh_cmd("140.44.4.11", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole status")
+print(out.strip())
+ok, out, err = run_ssh_cmd("140.44.4.11", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- sqlite3 /etc/pihole/pihole-FTL.db 'SELECT count(*) FROM queries WHERE timestamp > strftime(\"%s\",\"now\",\"-30 minutes\");'")
+print(f"Queries in last 30 min: {out.strip()}")
+ok, out, err = run_ssh_cmd("140.44.4.11", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- sqlite3 /etc/pihole/pihole-FTL.db 'SELECT client, count(*) FROM queries WHERE timestamp > strftime(\"%s\",\"now\",\"-30 minutes\") GROUP BY client ORDER BY count(*) DESC LIMIT 5;'")
+print(f"Top clients (last 30m):\n{out.strip()}")
+
+print("\n=== 2. Spaceport Pi-hole Query Flow ===")
+ok, out, err = run_ssh_cmd("140.44.4.41", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole status")
+print(out.strip())
+ok, out, err = run_ssh_cmd("140.44.4.41", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- sqlite3 /etc/pihole/pihole-FTL.db 'SELECT count(*) FROM queries WHERE timestamp > strftime(\"%s\",\"now\",\"-30 minutes\");'")
+print(f"Queries in last 30 min: {out.strip()}")
+ok, out, err = run_ssh_cmd("140.44.4.41", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- sqlite3 /etc/pihole/pihole-FTL.db 'SELECT client, count(*) FROM queries WHERE timestamp > strftime(\"%s\",\"now\",\"-30 minutes\") GROUP BY client ORDER BY count(*) DESC LIMIT 5;'")
+print(f"Top clients (last 30m):\n{out.strip()}")
+

scripts/patch_traffic_policy.py

@@ -0,0 +1,13 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl patch svc -n dns pihole-dns-vip --type=merge -p "{\\"spec\\":{\\"externalTrafficPolicy\\":\\"Local\\"}}"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Patch Aerodrome:")
+print(stdout.read().decode())
+print(stderr.read().decode())
+client.close()
+

scripts/restore_traffic_policy.py

@@ -0,0 +1,13 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'k3s kubectl patch svc -n dns pihole-dns-vip --type=merge -p "{\\"spec\\":{\\"externalTrafficPolicy\\":\\"Cluster\\"}}"'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("Restore Aerodrome:")
+print(stdout.read().decode())
+print(stderr.read().decode())
+client.close()
+

scripts/test_dns_and_pihole_live.py

@@ -0,0 +1,38 @@
+import paramiko
+
+SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def run_ssh_cmd(host, user, cmd, timeout=10):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(host, username=user, key_filename=SSH_KEY, timeout=timeout)
+        stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
+        out = stdout.read().decode('utf-8', errors='replace')
+        err = stderr.read().decode('utf-8', errors='replace')
+        client.close()
+        return True, out, err
+    except Exception as e:
+        return False, "", str(e)
+
+print("=== 1. Testing DNS resolution from nutserver (14.4.4.44, VLAN 50) ===")
+for vip, name in [("140.44.4.36", "Aerodrome"), ("140.44.4.65", "Spaceport")]:
+    ok, out, err = run_ssh_cmd("14.4.4.44", "root", f"nslookup snarfhome.snarfnet.net {vip}")
+    print(f"[{name} - {vip}]:\n{out.strip()}\n")
+
+print("=== 2. Testing NTP Peer status from nutserver (14.4.4.44, VLAN 50) ===")
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "chronyc sources")
+print(out.strip())
+
+print("\n=== 3. Checking Pi-hole Live Query Flow (Aerodrome pod) ===")
+ok, out, err = run_ssh_cmd("140.44.4.11", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole status")
+print(f"Aerodrome Pi-hole Status:\n{out.strip()}")
+ok, out, err = run_ssh_cmd("140.44.4.11", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- sqlite3 /etc/pihole/pihole-FTL.db 'SELECT client, count(*) FROM queries WHERE timestamp > strftime(\"%s\",\"now\",\"-10 minutes\") GROUP BY client ORDER BY count(*) DESC LIMIT 5;'")
+print(f"Top recent clients on Aerodrome (last 10m):\n{out.strip()}")
+
+print("\n=== 4. Checking Pi-hole Live Query Flow (Spaceport pod) ===")
+ok, out, err = run_ssh_cmd("140.44.4.41", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- pihole status")
+print(f"Spaceport Pi-hole Status:\n{out.strip()}")
+ok, out, err = run_ssh_cmd("140.44.4.41", "root", "k3s kubectl exec -n dns deploy/pihole -c pihole -- sqlite3 /etc/pihole/pihole-FTL.db 'SELECT client, count(*) FROM queries WHERE timestamp > strftime(\"%s\",\"now\",\"-10 minutes\") GROUP BY client ORDER BY count(*) DESC LIMIT 5;'")
+print(f"Top recent clients on Spaceport (last 10m):\n{out.strip()}")
+

scripts/test_firewall_and_dns.py

@@ -0,0 +1,47 @@
+import paramiko
+import time
+import socket
+
+SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def run_ssh_cmd(host, user, cmd, timeout=10):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(host, username=user, key_filename=SSH_KEY, timeout=timeout)
+        stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
+        out = stdout.read().decode('utf-8', errors='replace')
+        err = stderr.read().decode('utf-8', errors='replace')
+        client.close()
+        return True, out, err
+    except Exception as e:
+        return False, "", str(e)
+
+print("=== 1. Testing from nutserver (14.4.4.44, VLAN 50 Smart Home) ===")
+
+# Test DNS resolution to Aerodrome VIP (140.44.4.36)
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "dig @140.44.4.36 snarfhome.snarfnet.net +short +time=2 +tries=1")
+print(f"DNS query to 140.44.4.36: {'SUCCESS: ' + out.strip() if ok and out.strip() else 'FAILED: ' + err}")
+
+# Test DNS resolution to Spaceport VIP (140.44.4.65)
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "dig @140.44.4.65 snarfhome.snarfnet.net +short +time=2 +tries=1")
+print(f"DNS query to 140.44.4.65: {'SUCCESS: ' + out.strip() if ok and out.strip() else 'FAILED: ' + err}")
+
+# Test NTP connectivity to Aerodrome Master (140.44.4.11)
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "chronyc sources -v 2>&1")
+print(f"NTP query to 140.44.4.11:\n{out.strip()[:300]}")
+
+# Test BLOCKING: nutserver (VLAN 50) -> Compute (140.44.4.11:22)
+print("\n=== 2. Testing Firewall Drop: Smart Home (14.4.4.44) -> Compute (140.44.4.11:22) ===")
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "nc -z -w 3 140.44.4.11 22; echo ExitCode=$?")
+print(f"Connection test 140.44.4.11:22: {out.strip() or err.strip()}")
+
+# Test BLOCKING: nutserver (VLAN 50) -> Compute (140.44.4.71:22)
+print("=== 3. Testing Firewall Drop: Smart Home (14.4.4.44) -> Compute (140.44.4.71:22) ===")
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "nc -z -w 3 140.44.4.71 22; echo ExitCode=$?")
+print(f"Connection test 140.44.4.71:22: {out.strip() or err.strip()}")
+
+# Test ALLOW: nutserver (VLAN 50) -> OpenCanary Honeypot (14.4.4.99:22) (Intra-VLAN 50)
+print("\n=== 4. Testing Honeypot reachable from Smart Home: 14.4.4.44 -> 14.4.4.99:22 ===")
+ok, out, err = run_ssh_cmd("14.4.4.44", "root", "nc -z -w 3 14.4.4.99 22; echo ExitCode=$?")
+print(f"Honeypot reachability test: {out.strip() or err.strip()}")