import paramiko import sys import json sys.stdout.reconfigure(encoding='utf-8') c = paramiko.SSHClient() c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) def run_cmd(cmd): print(f"\n>>> {cmd}") _, stdout, stderr = c.exec_command(cmd) out = stdout.read().decode().strip() err = stderr.read().decode().strip() if out: print(out) if err: print("[stderr]", err) return out pod = run_cmd("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") print(f"Targeting Uptime Kuma Pod: {pod}") setup_script = '''import json import subprocess def query_db(sql): cmd = ["mariadb", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-e", sql] res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if res.returncode != 0: print("DB ERROR:", res.stderr) return res.stdout # 1. Take a full MariaDB dump before recovery subprocess.run(["mysqldump", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-r", "/app/data/pre_recovery_dump.sql"]) print("Created pre-recovery backup at /app/data/pre_recovery_dump.sql") # 2. Ensure Gotify Notification Exists notif_config = { "name": "Gotify Infrastructure Alerts", "type": "gotify", "isDefault": True, "gotifyServerUrl": "http://140.44.4.70:8090", "gotifyApplicationToken": "A7zBx9InFrAaLeRtStOk", "gotifyPriority": 8 } notif_json = json.dumps(notif_config) check_notif = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';") if "id" not in check_notif or not check_notif.split(): print("Creating Gotify Notification in Uptime Kuma...") escaped_json = notif_json.replace("'", "''") query_db(f"INSERT INTO notification (name, active, user_id, is_default, config) VALUES ('Gotify Infrastructure Alerts', 1, 1, 1, '{escaped_json}');") notif_id_out = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';") notif_id = [line.strip() for line in notif_id_out.splitlines() if line.strip().isdigit()][0] print(f"Gotify Notification ID: {notif_id}") # 3. Clean any broken previous monitors query_db("DELETE FROM monitor WHERE name IN ('Harvester Console (DNS)', 'Jenkins CI/CD (DNS)', 'NUT Master Daemon (TCP)');") # 4. Define Complete Fleet Monitors List (DNS + Direct VIP/IP) monitors = [ # --- DNS INGRESS MONITORS --- {"name": "Gotify Hub (DNS)", "type": "http", "url": "https://gotify.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Heimdall Portal (DNS)", "type": "http", "url": "https://portal.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "NPM Admin Console (DNS)", "type": "http", "url": "https://nginx.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "Authentik SSO (DNS)", "type": "http", "url": "https://authentik.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-399"]'}, {"name": "Rancher Management (DNS)", "type": "http", "url": "https://aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "Harvester Console (DNS)", "type": "http", "url": "https://harvester.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "Gitea DevOps (DNS)", "type": "http", "url": "https://git.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "ArgoCD GitOps (DNS)", "type": "http", "url": "https://argo.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "AWX Automation (DNS)", "type": "http", "url": "https://awx.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "Semaphore Ansible (DNS)", "type": "http", "url": "https://semaphore.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Jenkins CI/CD (DNS)", "type": "http", "url": "https://jenkins.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-403"]'}, {"name": "Zabbix Web UI (DNS)", "type": "http", "url": "https://zabbix.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Grafana Dashboards (DNS)", "type": "http", "url": "https://grafana.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Longhorn Aerodrome (DNS)", "type": "http", "url": "https://longhorn.aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Longhorn Spaceport (DNS)", "type": "http", "url": "https://longhorn.spaceport.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Home Assistant (DNS)", "type": "http", "url": "https://snarfhome.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "PeaNUT UPS Portal (DNS)", "type": "http", "url": "https://peanut.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "Open WebUI AI (DNS)", "type": "http", "url": "https://ai.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "SnarfNAS DSM (DNS)", "type": "http", "url": "https://snarfnas.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "SN-NAS DSM (DNS)", "type": "http", "url": "https://sn-nas.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "Homepage (DNS)", "type": "http", "url": "https://homepage.snarfnet.net", "interval": 60, "ignore_tls": 0}, {"name": "MeshCentral (DNS)", "type": "http", "url": "https://mesh.snarfnet.net", "interval": 60, "ignore_tls": 1}, {"name": "Homebox (DNS)", "type": "http", "url": "https://homebox.snarfnet.net", "interval": 60, "ignore_tls": 0}, # --- DIRECT VIP & IP MONITORS --- {"name": "Swarm VIP - Gotify (VIP)", "type": "http", "url": "http://140.44.4.70:8090", "interval": 30, "ignore_tls": 0}, {"name": "Swarm VIP - Heimdall (VIP)", "type": "http", "url": "http://140.44.4.70:85", "interval": 30, "ignore_tls": 0}, {"name": "Swarm VIP - Homebox (VIP)", "type": "http", "url": "http://140.44.4.70:3100", "interval": 30, "ignore_tls": 0}, {"name": "Swarm VIP - PeaNUT (VIP)", "type": "http", "url": "http://140.44.4.70:8050", "interval": 30, "ignore_tls": 0}, {"name": "Swarm Host - NPM (VIP)", "type": "http", "url": "http://140.44.4.74:81", "interval": 30, "ignore_tls": 0}, {"name": "Aerodrome VIP - Authentik (VIP)", "type": "http", "url": "https://140.44.4.29:444", "interval": 30, "ignore_tls": 1, "status_codes": '["200-399"]'}, {"name": "Aerodrome VIP - AWX (VIP)", "type": "http", "url": "https://140.44.4.30:443", "interval": 30, "ignore_tls": 1}, {"name": "Aerodrome VIP - Grafana (VIP)", "type": "http", "url": "http://140.44.4.32:88", "interval": 30, "ignore_tls": 0}, {"name": "Aerodrome VIP - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.36:80", "interval": 30, "ignore_tls": 0}, {"name": "Aerodrome VIP - Traefik (VIP)", "type": "http", "url": "https://140.44.4.34:443", "interval": 30, "ignore_tls": 1}, {"name": "Spaceport VIP - Zabbix Web (VIP)", "type": "http", "url": "http://140.44.4.57:8888", "interval": 30, "ignore_tls": 0}, {"name": "Spaceport VIP - Semaphore (VIP)", "type": "http", "url": "http://140.44.4.58:100", "interval": 30, "ignore_tls": 0}, {"name": "Spaceport VIP - Jenkins (VIP)", "type": "http", "url": "http://140.44.4.59:101", "interval": 30, "ignore_tls": 0}, {"name": "Spaceport VIP - Homepage (VIP)", "type": "http", "url": "http://140.44.4.60:8040", "interval": 30, "ignore_tls": 0}, {"name": "Spaceport VIP - Gitea (VIP)", "type": "http", "url": "http://140.44.4.62:90", "interval": 30, "ignore_tls": 0}, {"name": "Spaceport Node - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.41:80", "interval": 30, "ignore_tls": 0}, {"name": "Home Assistant Core (IP)", "type": "http", "url": "http://14.4.4.20:8123", "interval": 30, "ignore_tls": 0}, {"name": "AI Hailo Engine (IP)", "type": "http", "url": "http://140.44.4.7:8000/docs", "interval": 30, "ignore_tls": 0}, {"name": "AI Open WebUI (IP)", "type": "http", "url": "http://140.44.4.8:8080", "interval": 30, "ignore_tls": 0}, {"name": "Harvester HCI Console (VIP)", "type": "http", "url": "https://140.44.4.90:443", "interval": 30, "ignore_tls": 1}, {"name": "Synology NAS 1 DSM (IP)", "type": "http", "url": "https://14.10.10.71:5001", "interval": 30, "ignore_tls": 1}, {"name": "SnarfNAS DSM (IP)", "type": "http", "url": "https://14.10.10.72:5001", "interval": 30, "ignore_tls": 1}, {"name": "PeaNUT UPS Daemon (VIP)", "type": "http", "url": "http://140.44.4.70:8050/api/v1/devices", "interval": 30, "ignore_tls": 0}, ] # Fetch existing monitors to prevent duplicates existing_out = query_db("SELECT id, name FROM monitor;") existing_names = set() for line in existing_out.splitlines()[1:]: parts = line.split("\\t") if len(parts) >= 2: existing_names.add(parts[1].strip()) added_count = 0 for m in monitors: name = m["name"] if name in existing_names: print(f"Skipping existing monitor: {name}") continue m_type = m["type"] interval = m.get("interval", 60) ignore_tls = m.get("ignore_tls", 0) status_codes = m.get("status_codes", '["200-299"]') if m_type == "http": url = m["url"] sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, url, ignore_tls, accepted_statuscodes_json) VALUES ('{name}', 'http', 1, 1, {interval}, '{url}', {ignore_tls}, '{status_codes}');""" elif m_type == "port": host = m["hostname"] p = m["port"] sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, hostname, port) VALUES ('{name}', 'port', 1, 1, {interval}, '{host}', {p});""" query_db(sql) added_count += 1 print(f"Added monitor: {name}") # Link all monitors to Gotify notification all_monitors_out = query_db("SELECT id FROM monitor;") for line in all_monitors_out.splitlines()[1:]: line = line.strip() if line.isdigit(): m_id = line check_link = query_db(f"SELECT id FROM monitor_notification WHERE monitor_id = {m_id} AND notification_id = {notif_id};") if "id" not in check_link or not check_link.split(): query_db(f"INSERT INTO monitor_notification (monitor_id, notification_id) VALUES ({m_id}, {notif_id});") total_count = query_db("SELECT count(*) FROM monitor;") print(f"\\nRecovery Complete! Added {added_count} monitors. Total active monitors in DB: {total_count.strip()}") ''' stdin, _, _ = c.exec_command("cat > /tmp/recover_kuma.py") stdin.write(setup_script) stdin.close() run_cmd(f"kubectl cp /tmp/recover_kuma.py monitoring/{pod}:/tmp/recover_kuma.py") run_cmd(f"kubectl exec -n monitoring {pod} -- python3 /tmp/recover_kuma.py") print("\n=== Restarting Uptime Kuma Deployment ===") run_cmd("kubectl rollout restart deployment/uptime-kuma-2-1756668961 -n monitoring") run_cmd("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=90s") c.close() print("\n=== Recovery Successfully Finished ===")