feat: add administrative and deployment scripts for infrastructure management

scripts/authorize_and_test.py

@@ -0,0 +1,35 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+pub_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6ycMke4AaV2qzWfULZCTZOKhSKf4pY2qRGYjYzs1MGKutJxt2jkYNK2aS0XffxLRR49v23zHbBwLYraNl2Mkb9rsKsyvKn8CAa1CEl9NxOci4LDRT8CeWHRnMZn8Mdu1pfeRwM26/Bjnd4RJrSKC7fyM0OlOpEp0DgnuPBqBtPMFGw+TCUT8eZ0pBrAI94wEan4l35kmuBcqm6RH3KuD1dDJKnPLzJ3/T21CViYL0GgFIDDor/hGI45VfOI/jKmkgM4YoudTCde6uPV5e2yaJpORZlR4nDxg9ntXp8HxbHGZkXYr55Ibsjem0lMU6nBh33jGbJVvat1Ay4oNCbygJsHfC/8ppe0bnLhnasxVAXhTvklZZL/sf7K/VS0ZifpfqB6DKtr1lziOm2wQeahv+Y1uoHkVNa3XY8Fx7xGPeT1BGd9lfcbUoweoTdMPMIPaSZlNGY32AHYkM4PShG9yI1coAqXzMby0+Z/Qwtr4l6cLKVvcGgqh5sd+z1+YvZbs= root@SNINFR2LNXCTL01"
+
+# 1. Authorize on 140.44.4.71
+c71 = paramiko.SSHClient()
+c71.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c71.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+cmd = f"""
+mkdir -p ~/.ssh && chmod 700 ~/.ssh
+grep -qF '{pub_key}' ~/.ssh/authorized_keys 2>/dev/null || echo '{pub_key}' >> ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+"""
+c71.exec_command(cmd)
+c71.close()
+print("Authorized 140.44.4.11 key on 140.44.4.71.")
+
+# 2. Test SSH from 140.44.4.11 to 140.44.4.71
+c11 = paramiko.SSHClient()
+c11.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c11.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c11.exec_command("ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.71 hostname")
+out = stdout.read().decode('utf-8').strip()
+err = stderr.read().decode('utf-8').strip()
+print("Result of ssh from 140.44.4.11 to 140.44.4.71:")
+print("STDOUT:", out)
+if err: print("STDERR:", err)
+c11.close()
+

scripts/execute_housekeeping.py

@@ -0,0 +1,55 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def run_ssh(host, cmds, desc):
+    print(f"\n=======================================================")
+    print(f"[{desc}] Connecting to {host}...")
+    print(f"=======================================================")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=15)
+    for cmd in cmds:
+        print(f"\n>>> Running on {host}: {cmd}")
+        stdin, stdout, stderr = c.exec_command(cmd)
+        out = stdout.read().decode('utf-8')
+        err = stderr.read().decode('utf-8')
+        if out: print(out.strip())
+        if err: print("ERR:", err.strip())
+    c.close()
+
+# 1. Aerodrome Master (140.44.4.11)
+aero_cmds = [
+    # Take post-upgrade etcd snapshot
+    "k3s etcd-snapshot save --name post-upgrade-v2.15.3-final",
+    # List snapshots to verify
+    "k3s etcd-snapshot ls | tail -n 5",
+    # Clean up temporary rancher dirs in /tmp
+    "rm -rf /tmp/rancher-* /tmp/rancher_upgrade* /tmp/chart* /tmp/extracted*",
+    # Prune completed helm operations in cattle-system
+    "k3s kubectl -n cattle-system delete pod -l helm.cattle.io/chart=rancher --field-selector status.phase=Succeeded 2>/dev/null || true",
+    "k3s kubectl -n cattle-system delete pod -l app.kubernetes.io/managed-by=Helm --field-selector status.phase=Succeeded 2>/dev/null || true",
+    # Check cattle-system pods
+    "k3s kubectl -n cattle-system get pods"
+]
+
+run_ssh("140.44.4.11", aero_cmds, "Aerodrome Master")
+
+# 2. Spaceport Master (140.44.4.41)
+spaceport_cmds = [
+    # Take post-upgrade etcd snapshot
+    "k3s etcd-snapshot save --name post-upgrade-spaceport-final",
+    # List snapshots to verify
+    "k3s etcd-snapshot ls | tail -n 5",
+    # Prune completed system agent upgrader and helm pods
+    "k3s kubectl -n cattle-system delete pod -l app=system-agent-upgrader --field-selector status.phase=Succeeded 2>/dev/null || true",
+    "k3s kubectl -n cattle-system delete pod --field-selector status.phase=Succeeded 2>/dev/null || true",
+    # Check cattle-system pods
+    "k3s kubectl -n cattle-system get pods"
+]
+
+run_ssh("140.44.4.41", spaceport_cmds, "Spaceport Master")
+

scripts/find_dns_files.py

@@ -0,0 +1,13 @@
+import os
+import glob
+
+matches = []
+for root, dirs, files in os.walk(r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode"):
+    if ".git" in root: continue
+    for f in files:
+        low = f.lower()
+        if any(k in low for k in ["pihole", "technitium", "dns", "unbound", "honeypot", "firewall", "vlan", "network"]):
+            matches.append(os.path.join(root, f))
+
+for m in matches[:30]:
+    print(m)

scripts/investigate_dns_stubs.py

@@ -0,0 +1,33 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def check(host, name):
+    print(f"\n=================== {name} ({host}) ===================")
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=10)
+    
+    print("\n--- Namespaces ---")
+    stdin, stdout, stderr = c.exec_command("k3s kubectl get ns")
+    print(stdout.read().decode('utf-8'))
+    
+    print("\n--- Services with port 53 or pihole/technitium/dns in all namespaces ---")
+    stdin, stdout, stderr = c.exec_command("k3s kubectl get svc -A | grep -E '53|dns|pihole|technitium'")
+    print(stdout.read().decode('utf-8'))
+    
+    print("\n--- Deployments / StatefulSets with dns/pihole/technitium ---")
+    stdin, stdout, stderr = c.exec_command("k3s kubectl get deploy,sts,ds -A | grep -E 'pihole|technitium|unbound|dns'")
+    print(stdout.read().decode('utf-8'))
+    
+    print("\n--- MetalLB IPAddressPool or Services with LoadBalancer ---")
+    stdin, stdout, stderr = c.exec_command("k3s kubectl get ipaddresspool -A 2>/dev/null; k3s kubectl get svc -A --field-selector spec.type=LoadBalancer")
+    print(stdout.read().decode('utf-8'))
+    
+    c.close()
+
+check("140.44.4.11", "Aerodrome")
+check("140.44.4.41", "Spaceport")

scripts/read_step.py

@@ -4,8 +4,8 @@ with open(r"C:\Users\Snarf\.gemini\antigravity\brain\67541ce0-c72c-4983-b7f8-7f2
     for line in f:
         data = json.loads(line)
         idx = data.get("step_index", 0)
-        if 1970 <= idx <= 2010 and data.get("source") == "MODEL" and data.get("type") == "PLANNER_RESPONSE":
+        if 1973 <= idx <= 2050:
             content = data.get("content")
-            if content:
-                print(f"=== Step {idx} ===")
-                print(content[:800])
+            if content and len(content) > 100:
+                print(f"=== Step {idx} ({data.get('type')}) ===")
+                print(content[:400])

scripts/setup_cron_key.py

@@ -0,0 +1,16 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c.exec_command("cat ~/.ssh/id_rsa.pub || cat ~/.ssh/id_ed25519.pub || ssh-keygen -t ed25519 -N '' -f ~/.ssh/id_ed25519 && cat ~/.ssh/id_ed25519.pub")
+pub = stdout.read().decode('utf-8').strip()
+print("140.44.4.11 root pubkey:")
+print(pub)
+c.close()
+

scripts/setup_tls_sync.py

@@ -0,0 +1,76 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+sync_script = """#!/bin/bash
+set -euo pipefail
+
+LOG_FILE="/var/log/sync-rancher-tls.log"
+REMOTE_HOST="140.44.4.71"
+NPM_CERT_DIR="/data/glusterfs/swarm/brick/legacy_volumes/nginx/letsencrypt/live/npm-8"
+TMP_DIR="/tmp/rancher-tls-sync.$$"
+
+mkdir -p "$TMP_DIR"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+echo "[$(date -Is)] Checking Let's Encrypt certificate from $REMOTE_HOST..." >> "$LOG_FILE"
+
+# Fetch cert and key from NPM
+scp -o BatchMode=yes -o StrictHostKeyChecking=no "root@${REMOTE_HOST}:${NPM_CERT_DIR}/fullchain.pem" "$TMP_DIR/fullchain.pem" >> "$LOG_FILE" 2>&1
+scp -o BatchMode=yes -o StrictHostKeyChecking=no "root@${REMOTE_HOST}:${NPM_CERT_DIR}/privkey.pem" "$TMP_DIR/privkey.pem" >> "$LOG_FILE" 2>&1
+
+if [ ! -s "$TMP_DIR/fullchain.pem" ] || [ ! -s "$TMP_DIR/privkey.pem" ]; then
+    echo "[$(date -Is)] ERROR: Failed to retrieve certificate or key from $REMOTE_HOST" >> "$LOG_FILE"
+    exit 1
+fi
+
+# Compare fingerprints
+NEW_FP=$(openssl x509 -in "$TMP_DIR/fullchain.pem" -noout -fingerprint -sha256 2>/dev/null || echo "new_err")
+CURR_FP=$(k3s kubectl -n cattle-system get secret tls-rancher-ingress -o jsonpath='{.data.tls\\.crt}' 2>/dev/null | base64 -d | openssl x509 -noout -fingerprint -sha256 2>/dev/null || echo "curr_err")
+
+if [ "$NEW_FP" = "$CURR_FP" ]; then
+    echo "[$(date -Is)] Certificate is unchanged. ($NEW_FP)" >> "$LOG_FILE"
+    exit 0
+fi
+
+echo "[$(date -Is)] Certificate update detected! Applying to secret tls-rancher-ingress..." >> "$LOG_FILE"
+k3s kubectl -n cattle-system create secret tls tls-rancher-ingress \
+  --cert="$TMP_DIR/fullchain.pem" \
+  --key="$TMP_DIR/privkey.pem" \
+  --dry-run=client -o yaml | k3s kubectl apply -f - >> "$LOG_FILE" 2>&1
+
+echo "[$(date -Is)] Successfully updated tls-rancher-ingress. New fingerprint: $NEW_FP" >> "$LOG_FILE"
+"""
+
+cmd = f"""
+cat << 'EOF' > /usr/local/bin/sync-rancher-tls.sh
+{sync_script}
+EOF
+chmod +x /usr/local/bin/sync-rancher-tls.sh
+
+# Run once to test
+/usr/local/bin/sync-rancher-tls.sh
+cat /var/log/sync-rancher-tls.log | tail -n 5
+
+# Create daily cron
+cat << 'EOF' > /etc/cron.daily/sync-rancher-tls
+#!/bin/bash
+/usr/local/bin/sync-rancher-tls.sh >/dev/null 2>&1
+EOF
+chmod +x /etc/cron.daily/sync-rancher-tls
+"""
+
+stdin, stdout, stderr = c.exec_command(cmd)
+print("=== SETUP OUTPUT ===")
+print(stdout.read().decode('utf-8'))
+err = stderr.read().decode('utf-8')
+if err: print("=== ERRORS ===", err)
+
+c.close()
+

scripts/test_idempotency.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c.exec_command("/usr/local/bin/sync-rancher-tls.sh && cat /var/log/sync-rancher-tls.log | tail -n 5")
+print(stdout.read().decode('utf-8'))
+c.close()
+

scripts/test_inter_ssh.py

@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c.exec_command("ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.11 hostname 2>&1")
+print("From 140.44.4.71 to 140.44.4.11:")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+
+c2 = paramiko.SSHClient()
+c2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c2.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c2.exec_command("ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.71 hostname 2>&1")
+print("From 140.44.4.11 to 140.44.4.71:")
+print(stdout.read().decode('utf-8'))
+
+c2.close()
+