chore: remove unused file

scripts/check_db_creds.py

@@ -0,0 +1,30 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("--- Testing authentik dump ---")
+cmd = "kubectl exec -n infra authentik-postgresql-0 -- env"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("--- Testing gitea env ---")
+cmd = "kubectl exec -n infrastructure gitea-postgresql-0 -- env"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+print("--- Testing zabbix env ---")
+cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- env"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_nfs_mounts.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+    cmd = "df -h | grep -i nfs || echo 'no nfs mounts'"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"[{name.upper()}] NFS mounts:\n{stdout.read().decode().strip()}")
+    ssh.close()
+

scripts/dump_k8s_databases.py

@@ -0,0 +1,65 @@
+import paramiko
+import sys
+import datetime
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+db_targets = [
+    {
+        "cluster": "aerodrome",
+        "ip": "140.44.4.11",
+        "namespace": "infra",
+        "pod": "authentik-postgresql-0",
+        "name": "authentik",
+        "dump_cmd": "kubectl exec -n infra authentik-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/authentik_db_{timestamp}.sql.gz"
+    },
+    {
+        "cluster": "aerodrome",
+        "ip": "140.44.4.11",
+        "namespace": "awx",
+        "pod": "awx-postgres-15-0",
+        "name": "awx",
+        "dump_cmd": "kubectl exec -n awx awx-postgres-15-0 -- pg_dumpall -U postgres | gzip > /var/backups/awx_db_{timestamp}.sql.gz"
+    },
+    {
+        "cluster": "spaceport",
+        "ip": "140.44.4.41",
+        "namespace": "infrastructure",
+        "pod": "gitea-postgresql-0",
+        "name": "gitea",
+        "dump_cmd": "kubectl exec -n infrastructure gitea-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/gitea_db_{timestamp}.sql.gz"
+    },
+    {
+        "cluster": "spaceport",
+        "ip": "140.44.4.41",
+        "namespace": "monitoring",
+        "pod": "zabbix-postgresql-0",
+        "name": "zabbix",
+        "dump_cmd": "kubectl exec -n monitoring zabbix-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/zabbix_db_{timestamp}.sql.gz"
+    }
+]
+
+for target in db_targets:
+    cluster = target["cluster"]
+    ip = target["ip"]
+    name = target["name"]
+    print(f"[{cluster.upper()}] Backing up {name} database...")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=15)
+    
+    cmd = f"mkdir -p /var/backups && {target['dump_cmd'].replace('{timestamp}', timestamp)}"
+    stdin, stdout, stderr = ssh.exec_command(cmd, timeout=300)
+    exit_status = stdout.channel.recv_exit_status()
+    err = stderr.read().decode().strip()
+    
+    if exit_status == 0:
+        check_cmd = f"ls -lh /var/backups/{name}_db_{timestamp}.sql.gz"
+        _, chk_out, _ = ssh.exec_command(check_cmd)
+        print(f"  SUCCESS: {chk_out.read().decode().strip()}")
+    else:
+        print(f"  FAILED (exit {exit_status}): {err}")
+        
+    ssh.close()

scripts/inspect_db_env.py

@@ -0,0 +1,29 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== AERODROME DB USERS/ENV ===")
+for ns, pod in [('infra', 'authentik-postgresql-0'), ('awx', 'awx-postgres-15-0')]:
+    cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
+
+ssh.close()
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== SPACEPORT DB USERS/ENV ===")
+for ns, pod in [('infrastructure', 'gitea-postgresql-0'), ('monitoring', 'zabbix-postgresql-0')]:
+    cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
+
+ssh.close()
+

scripts/inspect_db_pods.py

@@ -0,0 +1,19 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+    cmd = "kubectl get pods -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    lines = stdout.read().decode().splitlines()
+    print(f"=== {name.upper()} PODS ===")
+    for line in lines:
+        lower = line.lower()
+        if any(k in lower for k in ['postgres', 'mysql', 'authentik', 'gitea', 'zabbix', 'awx', 'semaphore']):
+            print("  " + line)
+    ssh.close()
+

scripts/inspect_keepalived_conf.py

@@ -23,3 +23,4 @@ def main():
 
 if __name__ == '__main__':
     main()
+

scripts/sync_backups_to_nas.py

@@ -0,0 +1,37 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+nodes = [
+    ("aerodrome", "140.44.4.11", ["authentik_clean.sql.gz", "awx_clean.sql.gz"]),
+    ("spaceport", "140.44.4.41", ["gitea_clean.sql.gz", "zabbix_clean.sql.gz"])
+]
+
+for name, ip, files in nodes:
+    print(f"[{name.upper()}] Mounting NAS and copying backups...")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    setup_cmd = """
+    mkdir -p /mnt/nas_backup
+    mount -t nfs -o vers=4 14.10.10.71:/volume1/docker /mnt/nas_backup || mount -t nfs 14.10.10.71:/volume1/docker /mnt/nas_backup
+    mkdir -p /mnt/nas_backup/backups/databases
+    """
+    stdin, stdout, stderr = ssh.exec_command(setup_cmd)
+    stdout.channel.recv_exit_status()
+    
+    for f in files:
+        cp_cmd = f"cp /var/backups/{f} /mnt/nas_backup/backups/databases/ && ls -lh /mnt/nas_backup/backups/databases/{f}"
+        stdin, stdout, stderr = ssh.exec_command(cp_cmd)
+        stdout.channel.recv_exit_status()
+        print(f"  Saved to NAS: {stdout.read().decode().strip()}")
+        
+    umount_cmd = "umount -f /mnt/nas_backup || true"
+    stdin, stdout, stderr = ssh.exec_command(umount_cmd)
+    stdout.channel.recv_exit_status()
+    ssh.close()
+
+print("All database backups verified mirrored to Synology NAS (14.10.10.71).")
+

scripts/test_authentik_dump.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U authentik authentik' | gzip > /var/backups/authentik_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print("STDERR:", stderr.read().decode())
+_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
+print("RESULT:", out.read().decode().strip())
+ssh.close()
+

scripts/test_clean_db_dumps.py

@@ -0,0 +1,40 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+# Authentik
+print("=== Backing up Authentik ===")
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/postgres-password) pg_dump -U postgres authentik' | gzip > /var/backups/authentik_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+err = stderr.read().decode().strip()
+if err: print("Authentik err:", err)
+_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
+print("Authentik size:", out.read().decode().strip())
+ssh.close()
+
+# Spaceport (Gitea & Zabbix)
+print("=== Backing up Gitea & Zabbix ===")
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = "mkdir -p /var/backups && kubectl exec -n infrastructure gitea-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U ugandanese baptiste' | gzip > /var/backups/gitea_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+err = stderr.read().decode().strip()
+if err: print("Gitea err:", err)
+_, out, _ = ssh.exec_command("ls -lh /var/backups/gitea_clean.sql.gz")
+print("Gitea size:", out.read().decode().strip())
+
+cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- sh -c 'PGPASSWORD=zabbix pg_dump -U zabbix zabbix' | gzip > /var/backups/zabbix_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+err = stderr.read().decode().strip()
+if err: print("Zabbix err:", err)
+_, out, _ = ssh.exec_command("ls -lh /var/backups/zabbix_clean.sql.gz")
+print("Zabbix size:", out.read().decode().strip())
+
+ssh.close()
+

scripts/test_etcd_snapshots.py

@@ -0,0 +1,35 @@
+import paramiko
+import sys
+import datetime
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+clusters = [
+    ("aerodrome", "140.44.4.11"),
+    ("spaceport", "140.44.4.41")
+]
+
+for name, ip in clusters:
+    print(f"[{name.upper()}] Connecting to {ip}...")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    cmd = f"k3s etcd-snapshot save --name pre-upgrade-{name}-{timestamp}"
+    print(f"[{name.upper()}] Executing: {cmd}")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    out = stdout.read().decode().strip()
+    err = stderr.read().decode().strip()
+    if out:
+        print(f"[{name.upper()}] STDOUT: {out}")
+    if err:
+        print(f"[{name.upper()}] STDERR: {err}")
+        
+    check_cmd = "k3s etcd-snapshot ls"
+    stdin, stdout, stderr = ssh.exec_command(check_cmd)
+    print(f"[{name.upper()}] Recent Snapshots:\n" + stdout.read().decode().strip())
+    
+    ssh.close()
+    print("-" * 50)

scripts/test_keepalived_failover.py

@@ -64,3 +64,4 @@ def main():
 
 if __name__ == '__main__':
     main()
+

scripts/test_worker_os_patch.py

@@ -0,0 +1,113 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+MASTER_IP = '140.44.4.11'
+WORKER_IP = '140.44.4.27'
+NODE_NAME = 'sninfr2lnxwrk07'
+KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+print(f"=== STEP 1: TEST ROLLING OS PATCH ON {NODE_NAME} ({WORKER_IP}) ===")
+
+# 1. Connect to master to check pods on node
+master_ssh = paramiko.SSHClient()
+master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
+
+print(f"[*] Checking active pods on {NODE_NAME}...")
+_, stdout, _ = master_ssh.exec_command(f"kubectl get pods -A --field-selector spec.nodeName={NODE_NAME}")
+print(stdout.read().decode())
+
+print(f"[*] Cordoning {NODE_NAME}...")
+_, stdout, stderr = master_ssh.exec_command(f"kubectl cordon {NODE_NAME}")
+print("  " + stdout.read().decode().strip())
+
+print(f"[*] Draining {NODE_NAME}...")
+_, stdout, stderr = master_ssh.exec_command(f"kubectl drain {NODE_NAME} --ignore-daemonsets --delete-emptydir-data --force --timeout=180s")
+print("  " + stdout.read().decode().strip())
+err = stderr.read().decode().strip()
+if err:
+    print("  (drain notice):", err)
+
+# 2. Connect to worker to perform apt-get update & dist-upgrade
+print(f"[*] Connecting directly to worker {WORKER_IP} for package upgrades...")
+worker_ssh = paramiko.SSHClient()
+worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=10)
+
+upgrade_cmd = "DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade -y && apt-get autoremove -y"
+print("[*] Running dist-upgrade (preserving configurations)...")
+stdin, stdout, stderr = worker_ssh.exec_command(upgrade_cmd)
+exit_status = stdout.channel.recv_exit_status()
+print(f"[*] dist-upgrade completed with exit code: {exit_status}")
+
+# 3. Check reboot requirement
+_, stdout, _ = worker_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
+reboot_needed = stdout.read().decode().strip() == 'yes'
+print(f"[*] Reboot required: {reboot_needed}")
+
+if reboot_needed:
+    print("[*] Rebooting worker node...")
+    try:
+        worker_ssh.exec_command("reboot")
+    except Exception:
+        pass
+    worker_ssh.close()
+    
+    print("[*] Waiting 30 seconds for host to reboot...")
+    time.sleep(30)
+    
+    # Poll SSH until online
+    reconnected = False
+    for attempt in range(25):
+        try:
+            worker_ssh = paramiko.SSHClient()
+            worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=6)
+            reconnected = True
+            print(f"[*] Worker reconnected successfully on attempt {attempt+1}!")
+            break
+        except Exception:
+            time.sleep(5)
+            
+    if not reconnected:
+        print("[!] ERROR: Worker node did not come back online within timeout!")
+        sys.exit(1)
+        
+    # Verify services
+    _, stdout, _ = worker_ssh.exec_command("systemctl is-active systemd-timesyncd || systemctl is-active chrony")
+    print(f"  TimeSync: {stdout.read().decode().strip()}")
+    _, stdout, _ = worker_ssh.exec_command("systemctl is-active zabbix-agent2")
+    print(f"  ZabbixAgent2: {stdout.read().decode().strip()}")
+    _, stdout, _ = worker_ssh.exec_command("systemctl is-active k3s-agent || systemctl is-active k3s")
+    print(f"  K3sAgent: {stdout.read().decode().strip()}")
+    worker_ssh.close()
+else:
+    worker_ssh.close()
+
+# 4. Uncordon node
+print(f"[*] Uncordoning {NODE_NAME}...")
+_, stdout, _ = master_ssh.exec_command(f"kubectl uncordon {NODE_NAME}")
+print("  " + stdout.read().decode().strip())
+
+# 5. Wait for node to be Ready
+print(f"[*] Verifying node status in Kubernetes...")
+for _ in range(15):
+    _, stdout, _ = master_ssh.exec_command(f"kubectl get node {NODE_NAME} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'")
+    status = stdout.read().decode().strip()
+    if status == "True":
+        print(f"[OK] Node {NODE_NAME} is fully Ready!")
+        break
+    time.sleep(4)
+
+# 6. Check Longhorn volume robustness
+print("[*] Settling Longhorn volume replicas...")
+time.sleep(10)
+_, stdout, _ = master_ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness")
+print(stdout.read().decode())
+
+master_ssh.close()
+print(f"[SUCCESS] Step 1 Test Worker OS Patch on {NODE_NAME} completed flawlessly!")
+

scripts/tune_longhorn_settings.py

@@ -31,3 +31,4 @@ def main():
 
 if __name__ == '__main__':
     main()
+

scripts/update_keepalived_healthcheck.py

@@ -47,3 +47,4 @@ def main():
 
 if __name__ == '__main__':
     main()
+