Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: remove unused file

Browse Source
This commit is contained in:
Snarf
2026-11-03 14:32:06 -05:00
parent 5a583d1db9
commit 42f669266e
14 changed files with 401 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
scripts/check_db_creds.py Normal file
View File

@@ -0,0 +1,30 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("--- Testing authentik dump ---")
cmd = "kubectl exec -n infra authentik-postgresql-0 -- env"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("--- Testing gitea env ---")
cmd = "kubectl exec -n infrastructure gitea-postgresql-0 -- env"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
print("--- Testing zabbix env ---")
cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- env"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

14
scripts/check_nfs_mounts.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
cmd = "df -h | grep -i nfs || echo 'no nfs mounts'"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(f"[{name.upper()}] NFS mounts:\n{stdout.read().decode().strip()}")
ssh.close()

65
scripts/dump_k8s_databases.py Normal file
View File

@@ -0,0 +1,65 @@
import paramiko
import sys
import datetime
sys.stdout.reconfigure(encoding='utf-8')
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
db_targets = [
{
"cluster": "aerodrome",
"ip": "140.44.4.11",
"namespace": "infra",
"pod": "authentik-postgresql-0",
"name": "authentik",
"dump_cmd": "kubectl exec -n infra authentik-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/authentik_db_{timestamp}.sql.gz"
},
{
"cluster": "aerodrome",
"ip": "140.44.4.11",
"namespace": "awx",
"pod": "awx-postgres-15-0",
"name": "awx",
"dump_cmd": "kubectl exec -n awx awx-postgres-15-0 -- pg_dumpall -U postgres | gzip > /var/backups/awx_db_{timestamp}.sql.gz"
},
{
"cluster": "spaceport",
"ip": "140.44.4.41",
"namespace": "infrastructure",
"pod": "gitea-postgresql-0",
"name": "gitea",
"dump_cmd": "kubectl exec -n infrastructure gitea-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/gitea_db_{timestamp}.sql.gz"
},
{
"cluster": "spaceport",
"ip": "140.44.4.41",
"namespace": "monitoring",
"pod": "zabbix-postgresql-0",
"name": "zabbix",
"dump_cmd": "kubectl exec -n monitoring zabbix-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/zabbix_db_{timestamp}.sql.gz"
}
]
for target in db_targets:
cluster = target["cluster"]
ip = target["ip"]
name = target["name"]
print(f"[{cluster.upper()}] Backing up {name} database...")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=15)
cmd = f"mkdir -p /var/backups && {target['dump_cmd'].replace('{timestamp}', timestamp)}"
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=300)
exit_status = stdout.channel.recv_exit_status()
err = stderr.read().decode().strip()
if exit_status == 0:
check_cmd = f"ls -lh /var/backups/{name}_db_{timestamp}.sql.gz"
_, chk_out, _ = ssh.exec_command(check_cmd)
print(f" SUCCESS: {chk_out.read().decode().strip()}")
else:
print(f" FAILED (exit {exit_status}): {err}")
ssh.close()

29
scripts/inspect_db_env.py Normal file
View File

@@ -0,0 +1,29 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("=== AERODROME DB USERS/ENV ===")
for ns, pod in [('infra', 'authentik-postgresql-0'), ('awx', 'awx-postgres-15-0')]:
cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
ssh.close()
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("=== SPACEPORT DB USERS/ENV ===")
for ns, pod in [('infrastructure', 'gitea-postgresql-0'), ('monitoring', 'zabbix-postgresql-0')]:
cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
ssh.close()

19
scripts/inspect_db_pods.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
cmd = "kubectl get pods -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase"
stdin, stdout, stderr = ssh.exec_command(cmd)
lines = stdout.read().decode().splitlines()
print(f"=== {name.upper()} PODS ===")
for line in lines:
lower = line.lower()
if any(k in lower for k in ['postgres', 'mysql', 'authentik', 'gitea', 'zabbix', 'awx', 'semaphore']):
print(" " + line)
ssh.close()

1
scripts/inspect_keepalived_conf.py
View File

@@ -23,3 +23,4 @@ def main():
if __name__ == '__main__':
main()

37
scripts/sync_backups_to_nas.py Normal file
View File

@@ -0,0 +1,37 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
nodes = [
("aerodrome", "140.44.4.11", ["authentik_clean.sql.gz", "awx_clean.sql.gz"]),
("spaceport", "140.44.4.41", ["gitea_clean.sql.gz", "zabbix_clean.sql.gz"])
]
for name, ip, files in nodes:
print(f"[{name.upper()}] Mounting NAS and copying backups...")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
setup_cmd = """
mkdir -p /mnt/nas_backup
mount -t nfs -o vers=4 14.10.10.71:/volume1/docker /mnt/nas_backup || mount -t nfs 14.10.10.71:/volume1/docker /mnt/nas_backup
mkdir -p /mnt/nas_backup/backups/databases
"""
stdin, stdout, stderr = ssh.exec_command(setup_cmd)
stdout.channel.recv_exit_status()
for f in files:
cp_cmd = f"cp /var/backups/{f} /mnt/nas_backup/backups/databases/ && ls -lh /mnt/nas_backup/backups/databases/{f}"
stdin, stdout, stderr = ssh.exec_command(cp_cmd)
stdout.channel.recv_exit_status()
print(f" Saved to NAS: {stdout.read().decode().strip()}")
umount_cmd = "umount -f /mnt/nas_backup || true"
stdin, stdout, stderr = ssh.exec_command(umount_cmd)
stdout.channel.recv_exit_status()
ssh.close()
print("All database backups verified mirrored to Synology NAS (14.10.10.71).")

15
scripts/test_authentik_dump.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U authentik authentik' | gzip > /var/backups/authentik_clean.sql.gz"
stdin, stdout, stderr = ssh.exec_command(cmd)
print("STDERR:", stderr.read().decode())
_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
print("RESULT:", out.read().decode().strip())
ssh.close()

40
scripts/test_clean_db_dumps.py Normal file
View File

@@ -0,0 +1,40 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
# Authentik
print("=== Backing up Authentik ===")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/postgres-password) pg_dump -U postgres authentik' | gzip > /var/backups/authentik_clean.sql.gz"
stdin, stdout, stderr = ssh.exec_command(cmd)
err = stderr.read().decode().strip()
if err: print("Authentik err:", err)
_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
print("Authentik size:", out.read().decode().strip())
ssh.close()
# Spaceport (Gitea & Zabbix)
print("=== Backing up Gitea & Zabbix ===")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "mkdir -p /var/backups && kubectl exec -n infrastructure gitea-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U ugandanese baptiste' | gzip > /var/backups/gitea_clean.sql.gz"
stdin, stdout, stderr = ssh.exec_command(cmd)
err = stderr.read().decode().strip()
if err: print("Gitea err:", err)
_, out, _ = ssh.exec_command("ls -lh /var/backups/gitea_clean.sql.gz")
print("Gitea size:", out.read().decode().strip())
cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- sh -c 'PGPASSWORD=zabbix pg_dump -U zabbix zabbix' | gzip > /var/backups/zabbix_clean.sql.gz"
stdin, stdout, stderr = ssh.exec_command(cmd)
err = stderr.read().decode().strip()
if err: print("Zabbix err:", err)
_, out, _ = ssh.exec_command("ls -lh /var/backups/zabbix_clean.sql.gz")
print("Zabbix size:", out.read().decode().strip())
ssh.close()

35
scripts/test_etcd_snapshots.py Normal file
View File

@@ -0,0 +1,35 @@
import paramiko
import sys
import datetime
sys.stdout.reconfigure(encoding='utf-8')
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
clusters = [
("aerodrome", "140.44.4.11"),
("spaceport", "140.44.4.41")
]
for name, ip in clusters:
print(f"[{name.upper()}] Connecting to {ip}...")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = f"k3s etcd-snapshot save --name pre-upgrade-{name}-{timestamp}"
print(f"[{name.upper()}] Executing: {cmd}")
stdin, stdout, stderr = ssh.exec_command(cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
if out:
print(f"[{name.upper()}] STDOUT: {out}")
if err:
print(f"[{name.upper()}] STDERR: {err}")
check_cmd = "k3s etcd-snapshot ls"
stdin, stdout, stderr = ssh.exec_command(check_cmd)
print(f"[{name.upper()}] Recent Snapshots:\n" + stdout.read().decode().strip())
ssh.close()
print("-" * 50)

1
scripts/test_keepalived_failover.py
View File

@@ -64,3 +64,4 @@ def main():
if __name__ == '__main__':
main()

113
scripts/test_worker_os_patch.py Normal file
View File

@@ -0,0 +1,113 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
MASTER_IP = '140.44.4.11'
WORKER_IP = '140.44.4.27'
NODE_NAME = 'sninfr2lnxwrk07'
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
print(f"=== STEP 1: TEST ROLLING OS PATCH ON {NODE_NAME} ({WORKER_IP}) ===")
# 1. Connect to master to check pods on node
master_ssh = paramiko.SSHClient()
master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
print(f"[*] Checking active pods on {NODE_NAME}...")
_, stdout, _ = master_ssh.exec_command(f"kubectl get pods -A --field-selector spec.nodeName={NODE_NAME}")
print(stdout.read().decode())
print(f"[*] Cordoning {NODE_NAME}...")
_, stdout, stderr = master_ssh.exec_command(f"kubectl cordon {NODE_NAME}")
print(" " + stdout.read().decode().strip())
print(f"[*] Draining {NODE_NAME}...")
_, stdout, stderr = master_ssh.exec_command(f"kubectl drain {NODE_NAME} --ignore-daemonsets --delete-emptydir-data --force --timeout=180s")
print(" " + stdout.read().decode().strip())
err = stderr.read().decode().strip()
if err:
print(" (drain notice):", err)
# 2. Connect to worker to perform apt-get update & dist-upgrade
print(f"[*] Connecting directly to worker {WORKER_IP} for package upgrades...")
worker_ssh = paramiko.SSHClient()
worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=10)
upgrade_cmd = "DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade -y && apt-get autoremove -y"
print("[*] Running dist-upgrade (preserving configurations)...")
stdin, stdout, stderr = worker_ssh.exec_command(upgrade_cmd)
exit_status = stdout.channel.recv_exit_status()
print(f"[*] dist-upgrade completed with exit code: {exit_status}")
# 3. Check reboot requirement
_, stdout, _ = worker_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
reboot_needed = stdout.read().decode().strip() == 'yes'
print(f"[*] Reboot required: {reboot_needed}")
if reboot_needed:
print("[*] Rebooting worker node...")
try:
worker_ssh.exec_command("reboot")
except Exception:
pass
worker_ssh.close()
print("[*] Waiting 30 seconds for host to reboot...")
time.sleep(30)
# Poll SSH until online
reconnected = False
for attempt in range(25):
try:
worker_ssh = paramiko.SSHClient()
worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=6)
reconnected = True
print(f"[*] Worker reconnected successfully on attempt {attempt+1}!")
break
except Exception:
time.sleep(5)
if not reconnected:
print("[!] ERROR: Worker node did not come back online within timeout!")
sys.exit(1)
# Verify services
_, stdout, _ = worker_ssh.exec_command("systemctl is-active systemd-timesyncd || systemctl is-active chrony")
print(f" TimeSync: {stdout.read().decode().strip()}")
_, stdout, _ = worker_ssh.exec_command("systemctl is-active zabbix-agent2")
print(f" ZabbixAgent2: {stdout.read().decode().strip()}")
_, stdout, _ = worker_ssh.exec_command("systemctl is-active k3s-agent || systemctl is-active k3s")
print(f" K3sAgent: {stdout.read().decode().strip()}")
worker_ssh.close()
else:
worker_ssh.close()
# 4. Uncordon node
print(f"[*] Uncordoning {NODE_NAME}...")
_, stdout, _ = master_ssh.exec_command(f"kubectl uncordon {NODE_NAME}")
print(" " + stdout.read().decode().strip())
# 5. Wait for node to be Ready
print(f"[*] Verifying node status in Kubernetes...")
for _ in range(15):
_, stdout, _ = master_ssh.exec_command(f"kubectl get node {NODE_NAME} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'")
status = stdout.read().decode().strip()
if status == "True":
print(f"[OK] Node {NODE_NAME} is fully Ready!")
break
time.sleep(4)
# 6. Check Longhorn volume robustness
print("[*] Settling Longhorn volume replicas...")
time.sleep(10)
_, stdout, _ = master_ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness")
print(stdout.read().decode())
master_ssh.close()
print(f"[SUCCESS] Step 1 Test Worker OS Patch on {NODE_NAME} completed flawlessly!")

1
scripts/tune_longhorn_settings.py
View File

@@ -31,3 +31,4 @@ def main():
if __name__ == '__main__':
main()

1
scripts/update_keepalived_healthcheck.py
View File

@@ -47,3 +47,4 @@ def main():
if __name__ == '__main__':
main()