From 42f669266e824dfef7ffd36b1eb6bb16c5876756 Mon Sep 17 00:00:00 2001 From: Snarf Date: Tue, 3 Nov 2026 14:32:06 -0500 Subject: [PATCH] chore: remove unused file --- scripts/check_db_creds.py | 30 ++++++ scripts/check_nfs_mounts.py | 14 +++ scripts/dump_k8s_databases.py | 65 +++++++++++++ scripts/inspect_db_env.py | 29 ++++++ scripts/inspect_db_pods.py | 19 ++++ scripts/inspect_keepalived_conf.py | 1 + scripts/sync_backups_to_nas.py | 37 ++++++++ scripts/test_authentik_dump.py | 15 +++ scripts/test_clean_db_dumps.py | 40 ++++++++ scripts/test_etcd_snapshots.py | 35 +++++++ scripts/test_keepalived_failover.py | 1 + scripts/test_worker_os_patch.py | 113 +++++++++++++++++++++++ scripts/tune_longhorn_settings.py | 1 + scripts/update_keepalived_healthcheck.py | 1 + 14 files changed, 401 insertions(+) create mode 100644 scripts/check_db_creds.py create mode 100644 scripts/check_nfs_mounts.py create mode 100644 scripts/dump_k8s_databases.py create mode 100644 scripts/inspect_db_env.py create mode 100644 scripts/inspect_db_pods.py create mode 100644 scripts/sync_backups_to_nas.py create mode 100644 scripts/test_authentik_dump.py create mode 100644 scripts/test_clean_db_dumps.py create mode 100644 scripts/test_etcd_snapshots.py create mode 100644 scripts/test_worker_os_patch.py diff --git a/scripts/check_db_creds.py b/scripts/check_db_creds.py new file mode 100644 index 0000000..659eb44 --- /dev/null +++ b/scripts/check_db_creds.py @@ -0,0 +1,30 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("--- Testing authentik dump ---") +cmd = "kubectl exec -n infra authentik-postgresql-0 -- env" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("--- Testing gitea env ---") +cmd = "kubectl exec -n infrastructure gitea-postgresql-0 -- env" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) + +print("--- Testing zabbix env ---") +cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- env" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/check_nfs_mounts.py b/scripts/check_nfs_mounts.py new file mode 100644 index 0000000..8d98dfd --- /dev/null +++ b/scripts/check_nfs_mounts.py @@ -0,0 +1,14 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + cmd = "df -h | grep -i nfs || echo 'no nfs mounts'" + stdin, stdout, stderr = ssh.exec_command(cmd) + print(f"[{name.upper()}] NFS mounts:\n{stdout.read().decode().strip()}") + ssh.close() + diff --git a/scripts/dump_k8s_databases.py b/scripts/dump_k8s_databases.py new file mode 100644 index 0000000..4e05c35 --- /dev/null +++ b/scripts/dump_k8s_databases.py @@ -0,0 +1,65 @@ +import paramiko +import sys +import datetime + +sys.stdout.reconfigure(encoding='utf-8') + +timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + +db_targets = [ + { + "cluster": "aerodrome", + "ip": "140.44.4.11", + "namespace": "infra", + "pod": "authentik-postgresql-0", + "name": "authentik", + "dump_cmd": "kubectl exec -n infra authentik-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/authentik_db_{timestamp}.sql.gz" + }, + { + "cluster": "aerodrome", + "ip": "140.44.4.11", + "namespace": "awx", + "pod": "awx-postgres-15-0", + "name": "awx", + "dump_cmd": "kubectl exec -n awx awx-postgres-15-0 -- pg_dumpall -U postgres | gzip > /var/backups/awx_db_{timestamp}.sql.gz" + }, + { + "cluster": "spaceport", + "ip": "140.44.4.41", + "namespace": "infrastructure", + "pod": "gitea-postgresql-0", + "name": "gitea", + "dump_cmd": "kubectl exec -n infrastructure gitea-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/gitea_db_{timestamp}.sql.gz" + }, + { + "cluster": "spaceport", + "ip": "140.44.4.41", + "namespace": "monitoring", + "pod": "zabbix-postgresql-0", + "name": "zabbix", + "dump_cmd": "kubectl exec -n monitoring zabbix-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/zabbix_db_{timestamp}.sql.gz" + } +] + +for target in db_targets: + cluster = target["cluster"] + ip = target["ip"] + name = target["name"] + print(f"[{cluster.upper()}] Backing up {name} database...") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=15) + + cmd = f"mkdir -p /var/backups && {target['dump_cmd'].replace('{timestamp}', timestamp)}" + stdin, stdout, stderr = ssh.exec_command(cmd, timeout=300) + exit_status = stdout.channel.recv_exit_status() + err = stderr.read().decode().strip() + + if exit_status == 0: + check_cmd = f"ls -lh /var/backups/{name}_db_{timestamp}.sql.gz" + _, chk_out, _ = ssh.exec_command(check_cmd) + print(f" SUCCESS: {chk_out.read().decode().strip()}") + else: + print(f" FAILED (exit {exit_status}): {err}") + + ssh.close() diff --git a/scripts/inspect_db_env.py b/scripts/inspect_db_env.py new file mode 100644 index 0000000..985ee45 --- /dev/null +++ b/scripts/inspect_db_env.py @@ -0,0 +1,29 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("=== AERODROME DB USERS/ENV ===") +for ns, pod in [('infra', 'authentik-postgresql-0'), ('awx', 'awx-postgres-15-0')]: + cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'" + stdin, stdout, stderr = ssh.exec_command(cmd) + print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}") + +ssh.close() + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("=== SPACEPORT DB USERS/ENV ===") +for ns, pod in [('infrastructure', 'gitea-postgresql-0'), ('monitoring', 'zabbix-postgresql-0')]: + cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'" + stdin, stdout, stderr = ssh.exec_command(cmd) + print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}") + +ssh.close() + diff --git a/scripts/inspect_db_pods.py b/scripts/inspect_db_pods.py new file mode 100644 index 0000000..a5f54d6 --- /dev/null +++ b/scripts/inspect_db_pods.py @@ -0,0 +1,19 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + cmd = "kubectl get pods -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase" + stdin, stdout, stderr = ssh.exec_command(cmd) + lines = stdout.read().decode().splitlines() + print(f"=== {name.upper()} PODS ===") + for line in lines: + lower = line.lower() + if any(k in lower for k in ['postgres', 'mysql', 'authentik', 'gitea', 'zabbix', 'awx', 'semaphore']): + print(" " + line) + ssh.close() + diff --git a/scripts/inspect_keepalived_conf.py b/scripts/inspect_keepalived_conf.py index 31fa4de..617ccbf 100644 --- a/scripts/inspect_keepalived_conf.py +++ b/scripts/inspect_keepalived_conf.py @@ -23,3 +23,4 @@ def main(): if __name__ == '__main__': main() + diff --git a/scripts/sync_backups_to_nas.py b/scripts/sync_backups_to_nas.py new file mode 100644 index 0000000..79a2216 --- /dev/null +++ b/scripts/sync_backups_to_nas.py @@ -0,0 +1,37 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +nodes = [ + ("aerodrome", "140.44.4.11", ["authentik_clean.sql.gz", "awx_clean.sql.gz"]), + ("spaceport", "140.44.4.41", ["gitea_clean.sql.gz", "zabbix_clean.sql.gz"]) +] + +for name, ip, files in nodes: + print(f"[{name.upper()}] Mounting NAS and copying backups...") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + setup_cmd = """ + mkdir -p /mnt/nas_backup + mount -t nfs -o vers=4 14.10.10.71:/volume1/docker /mnt/nas_backup || mount -t nfs 14.10.10.71:/volume1/docker /mnt/nas_backup + mkdir -p /mnt/nas_backup/backups/databases + """ + stdin, stdout, stderr = ssh.exec_command(setup_cmd) + stdout.channel.recv_exit_status() + + for f in files: + cp_cmd = f"cp /var/backups/{f} /mnt/nas_backup/backups/databases/ && ls -lh /mnt/nas_backup/backups/databases/{f}" + stdin, stdout, stderr = ssh.exec_command(cp_cmd) + stdout.channel.recv_exit_status() + print(f" Saved to NAS: {stdout.read().decode().strip()}") + + umount_cmd = "umount -f /mnt/nas_backup || true" + stdin, stdout, stderr = ssh.exec_command(umount_cmd) + stdout.channel.recv_exit_status() + ssh.close() + +print("All database backups verified mirrored to Synology NAS (14.10.10.71).") + diff --git a/scripts/test_authentik_dump.py b/scripts/test_authentik_dump.py new file mode 100644 index 0000000..10646d4 --- /dev/null +++ b/scripts/test_authentik_dump.py @@ -0,0 +1,15 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U authentik authentik' | gzip > /var/backups/authentik_clean.sql.gz" +stdin, stdout, stderr = ssh.exec_command(cmd) +print("STDERR:", stderr.read().decode()) +_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz") +print("RESULT:", out.read().decode().strip()) +ssh.close() + diff --git a/scripts/test_clean_db_dumps.py b/scripts/test_clean_db_dumps.py new file mode 100644 index 0000000..0be9d4b --- /dev/null +++ b/scripts/test_clean_db_dumps.py @@ -0,0 +1,40 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +# Authentik +print("=== Backing up Authentik ===") +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/postgres-password) pg_dump -U postgres authentik' | gzip > /var/backups/authentik_clean.sql.gz" +stdin, stdout, stderr = ssh.exec_command(cmd) +err = stderr.read().decode().strip() +if err: print("Authentik err:", err) +_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz") +print("Authentik size:", out.read().decode().strip()) +ssh.close() + +# Spaceport (Gitea & Zabbix) +print("=== Backing up Gitea & Zabbix ===") +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = "mkdir -p /var/backups && kubectl exec -n infrastructure gitea-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U ugandanese baptiste' | gzip > /var/backups/gitea_clean.sql.gz" +stdin, stdout, stderr = ssh.exec_command(cmd) +err = stderr.read().decode().strip() +if err: print("Gitea err:", err) +_, out, _ = ssh.exec_command("ls -lh /var/backups/gitea_clean.sql.gz") +print("Gitea size:", out.read().decode().strip()) + +cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- sh -c 'PGPASSWORD=zabbix pg_dump -U zabbix zabbix' | gzip > /var/backups/zabbix_clean.sql.gz" +stdin, stdout, stderr = ssh.exec_command(cmd) +err = stderr.read().decode().strip() +if err: print("Zabbix err:", err) +_, out, _ = ssh.exec_command("ls -lh /var/backups/zabbix_clean.sql.gz") +print("Zabbix size:", out.read().decode().strip()) + +ssh.close() + diff --git a/scripts/test_etcd_snapshots.py b/scripts/test_etcd_snapshots.py new file mode 100644 index 0000000..6a24216 --- /dev/null +++ b/scripts/test_etcd_snapshots.py @@ -0,0 +1,35 @@ +import paramiko +import sys +import datetime + +sys.stdout.reconfigure(encoding='utf-8') + +timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + +clusters = [ + ("aerodrome", "140.44.4.11"), + ("spaceport", "140.44.4.41") +] + +for name, ip in clusters: + print(f"[{name.upper()}] Connecting to {ip}...") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + cmd = f"k3s etcd-snapshot save --name pre-upgrade-{name}-{timestamp}" + print(f"[{name.upper()}] Executing: {cmd}") + stdin, stdout, stderr = ssh.exec_command(cmd) + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + if out: + print(f"[{name.upper()}] STDOUT: {out}") + if err: + print(f"[{name.upper()}] STDERR: {err}") + + check_cmd = "k3s etcd-snapshot ls" + stdin, stdout, stderr = ssh.exec_command(check_cmd) + print(f"[{name.upper()}] Recent Snapshots:\n" + stdout.read().decode().strip()) + + ssh.close() + print("-" * 50) diff --git a/scripts/test_keepalived_failover.py b/scripts/test_keepalived_failover.py index 7ecd9e8..3539abf 100644 --- a/scripts/test_keepalived_failover.py +++ b/scripts/test_keepalived_failover.py @@ -64,3 +64,4 @@ def main(): if __name__ == '__main__': main() + diff --git a/scripts/test_worker_os_patch.py b/scripts/test_worker_os_patch.py new file mode 100644 index 0000000..36cd5d8 --- /dev/null +++ b/scripts/test_worker_os_patch.py @@ -0,0 +1,113 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +WORKER_IP = '140.44.4.27' +NODE_NAME = 'sninfr2lnxwrk07' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +print(f"=== STEP 1: TEST ROLLING OS PATCH ON {NODE_NAME} ({WORKER_IP}) ===") + +# 1. Connect to master to check pods on node +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"[*] Checking active pods on {NODE_NAME}...") +_, stdout, _ = master_ssh.exec_command(f"kubectl get pods -A --field-selector spec.nodeName={NODE_NAME}") +print(stdout.read().decode()) + +print(f"[*] Cordoning {NODE_NAME}...") +_, stdout, stderr = master_ssh.exec_command(f"kubectl cordon {NODE_NAME}") +print(" " + stdout.read().decode().strip()) + +print(f"[*] Draining {NODE_NAME}...") +_, stdout, stderr = master_ssh.exec_command(f"kubectl drain {NODE_NAME} --ignore-daemonsets --delete-emptydir-data --force --timeout=180s") +print(" " + stdout.read().decode().strip()) +err = stderr.read().decode().strip() +if err: + print(" (drain notice):", err) + +# 2. Connect to worker to perform apt-get update & dist-upgrade +print(f"[*] Connecting directly to worker {WORKER_IP} for package upgrades...") +worker_ssh = paramiko.SSHClient() +worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +upgrade_cmd = "DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade -y && apt-get autoremove -y" +print("[*] Running dist-upgrade (preserving configurations)...") +stdin, stdout, stderr = worker_ssh.exec_command(upgrade_cmd) +exit_status = stdout.channel.recv_exit_status() +print(f"[*] dist-upgrade completed with exit code: {exit_status}") + +# 3. Check reboot requirement +_, stdout, _ = worker_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") +reboot_needed = stdout.read().decode().strip() == 'yes' +print(f"[*] Reboot required: {reboot_needed}") + +if reboot_needed: + print("[*] Rebooting worker node...") + try: + worker_ssh.exec_command("reboot") + except Exception: + pass + worker_ssh.close() + + print("[*] Waiting 30 seconds for host to reboot...") + time.sleep(30) + + # Poll SSH until online + reconnected = False + for attempt in range(25): + try: + worker_ssh = paramiko.SSHClient() + worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=6) + reconnected = True + print(f"[*] Worker reconnected successfully on attempt {attempt+1}!") + break + except Exception: + time.sleep(5) + + if not reconnected: + print("[!] ERROR: Worker node did not come back online within timeout!") + sys.exit(1) + + # Verify services + _, stdout, _ = worker_ssh.exec_command("systemctl is-active systemd-timesyncd || systemctl is-active chrony") + print(f" TimeSync: {stdout.read().decode().strip()}") + _, stdout, _ = worker_ssh.exec_command("systemctl is-active zabbix-agent2") + print(f" ZabbixAgent2: {stdout.read().decode().strip()}") + _, stdout, _ = worker_ssh.exec_command("systemctl is-active k3s-agent || systemctl is-active k3s") + print(f" K3sAgent: {stdout.read().decode().strip()}") + worker_ssh.close() +else: + worker_ssh.close() + +# 4. Uncordon node +print(f"[*] Uncordoning {NODE_NAME}...") +_, stdout, _ = master_ssh.exec_command(f"kubectl uncordon {NODE_NAME}") +print(" " + stdout.read().decode().strip()) + +# 5. Wait for node to be Ready +print(f"[*] Verifying node status in Kubernetes...") +for _ in range(15): + _, stdout, _ = master_ssh.exec_command(f"kubectl get node {NODE_NAME} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'") + status = stdout.read().decode().strip() + if status == "True": + print(f"[OK] Node {NODE_NAME} is fully Ready!") + break + time.sleep(4) + +# 6. Check Longhorn volume robustness +print("[*] Settling Longhorn volume replicas...") +time.sleep(10) +_, stdout, _ = master_ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness") +print(stdout.read().decode()) + +master_ssh.close() +print(f"[SUCCESS] Step 1 Test Worker OS Patch on {NODE_NAME} completed flawlessly!") + diff --git a/scripts/tune_longhorn_settings.py b/scripts/tune_longhorn_settings.py index 21a75cf..f04474d 100644 --- a/scripts/tune_longhorn_settings.py +++ b/scripts/tune_longhorn_settings.py @@ -31,3 +31,4 @@ def main(): if __name__ == '__main__': main() + diff --git a/scripts/update_keepalived_healthcheck.py b/scripts/update_keepalived_healthcheck.py index 3e645fd..cf8773a 100644 --- a/scripts/update_keepalived_healthcheck.py +++ b/scripts/update_keepalived_healthcheck.py @@ -47,3 +47,4 @@ def main(): if __name__ == '__main__': main() +