177 lines
11 KiB
Python
177 lines
11 KiB
Python
import paramiko
|
|
import sys
|
|
import json
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
c = paramiko.SSHClient()
|
|
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
|
|
|
def run_cmd(cmd):
|
|
print(f"\n>>> {cmd}")
|
|
_, stdout, stderr = c.exec_command(cmd)
|
|
out = stdout.read().decode().strip()
|
|
err = stderr.read().decode().strip()
|
|
if out:
|
|
print(out)
|
|
if err:
|
|
print("[stderr]", err)
|
|
return out
|
|
|
|
pod = run_cmd("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
|
print(f"Targeting Uptime Kuma Pod: {pod}")
|
|
|
|
setup_script = '''import json
|
|
import subprocess
|
|
|
|
def query_db(sql):
|
|
cmd = ["mariadb", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-e", sql]
|
|
res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
if res.returncode != 0:
|
|
print("DB ERROR:", res.stderr)
|
|
return res.stdout
|
|
|
|
# 1. Take a full MariaDB dump before recovery
|
|
subprocess.run(["mysqldump", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-r", "/app/data/pre_recovery_dump.sql"])
|
|
print("Created pre-recovery backup at /app/data/pre_recovery_dump.sql")
|
|
|
|
# 2. Ensure Gotify Notification Exists
|
|
notif_config = {
|
|
"name": "Gotify Infrastructure Alerts",
|
|
"type": "gotify",
|
|
"isDefault": True,
|
|
"gotifyServerUrl": "http://140.44.4.70:8090",
|
|
"gotifyApplicationToken": "A7zBx9InFrAaLeRtStOk",
|
|
"gotifyPriority": 8
|
|
}
|
|
notif_json = json.dumps(notif_config)
|
|
|
|
check_notif = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';")
|
|
if "id" not in check_notif or not check_notif.split():
|
|
print("Creating Gotify Notification in Uptime Kuma...")
|
|
escaped_json = notif_json.replace("'", "''")
|
|
query_db(f"INSERT INTO notification (name, active, user_id, is_default, config) VALUES ('Gotify Infrastructure Alerts', 1, 1, 1, '{escaped_json}');")
|
|
|
|
notif_id_out = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';")
|
|
notif_id = [line.strip() for line in notif_id_out.splitlines() if line.strip().isdigit()][0]
|
|
print(f"Gotify Notification ID: {notif_id}")
|
|
|
|
# 3. Clean any broken previous monitors
|
|
query_db("DELETE FROM monitor WHERE name IN ('Harvester Console (DNS)', 'Jenkins CI/CD (DNS)', 'NUT Master Daemon (TCP)');")
|
|
|
|
# 4. Define Complete Fleet Monitors List (DNS + Direct VIP/IP)
|
|
monitors = [
|
|
# --- DNS INGRESS MONITORS ---
|
|
{"name": "Gotify Hub (DNS)", "type": "http", "url": "https://gotify.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Heimdall Portal (DNS)", "type": "http", "url": "https://portal.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "NPM Admin Console (DNS)", "type": "http", "url": "https://nginx.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "Authentik SSO (DNS)", "type": "http", "url": "https://authentik.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-399"]'},
|
|
{"name": "Rancher Management (DNS)", "type": "http", "url": "https://aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "Harvester Console (DNS)", "type": "http", "url": "https://harvester.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "Gitea DevOps (DNS)", "type": "http", "url": "https://git.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "ArgoCD GitOps (DNS)", "type": "http", "url": "https://argo.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "AWX Automation (DNS)", "type": "http", "url": "https://awx.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "Semaphore Ansible (DNS)", "type": "http", "url": "https://semaphore.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Jenkins CI/CD (DNS)", "type": "http", "url": "https://jenkins.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-403"]'},
|
|
{"name": "Zabbix Web UI (DNS)", "type": "http", "url": "https://zabbix.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Grafana Dashboards (DNS)", "type": "http", "url": "https://grafana.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Longhorn Aerodrome (DNS)", "type": "http", "url": "https://longhorn.aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Longhorn Spaceport (DNS)", "type": "http", "url": "https://longhorn.spaceport.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Home Assistant (DNS)", "type": "http", "url": "https://snarfhome.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "PeaNUT UPS Portal (DNS)", "type": "http", "url": "https://peanut.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "Open WebUI AI (DNS)", "type": "http", "url": "https://ai.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "SnarfNAS DSM (DNS)", "type": "http", "url": "https://snarfnas.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "SN-NAS DSM (DNS)", "type": "http", "url": "https://sn-nas.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "Homepage (DNS)", "type": "http", "url": "https://homepage.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
{"name": "MeshCentral (DNS)", "type": "http", "url": "https://mesh.snarfnet.net", "interval": 60, "ignore_tls": 1},
|
|
{"name": "Homebox (DNS)", "type": "http", "url": "https://homebox.snarfnet.net", "interval": 60, "ignore_tls": 0},
|
|
|
|
# --- DIRECT VIP & IP MONITORS ---
|
|
{"name": "Swarm VIP - Gotify (VIP)", "type": "http", "url": "http://140.44.4.70:8090", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Swarm VIP - Heimdall (VIP)", "type": "http", "url": "http://140.44.4.70:85", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Swarm VIP - Homebox (VIP)", "type": "http", "url": "http://140.44.4.70:3100", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Swarm VIP - PeaNUT (VIP)", "type": "http", "url": "http://140.44.4.70:8050", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Swarm Host - NPM (VIP)", "type": "http", "url": "http://140.44.4.74:81", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Aerodrome VIP - Authentik (VIP)", "type": "http", "url": "https://140.44.4.29:444", "interval": 30, "ignore_tls": 1, "status_codes": '["200-399"]'},
|
|
{"name": "Aerodrome VIP - AWX (VIP)", "type": "http", "url": "https://140.44.4.30:443", "interval": 30, "ignore_tls": 1},
|
|
{"name": "Aerodrome VIP - Grafana (VIP)", "type": "http", "url": "http://140.44.4.32:88", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Aerodrome VIP - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.36:80", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Aerodrome VIP - Traefik (VIP)", "type": "http", "url": "https://140.44.4.34:443", "interval": 30, "ignore_tls": 1},
|
|
{"name": "Spaceport VIP - Zabbix Web (VIP)", "type": "http", "url": "http://140.44.4.57:8888", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Spaceport VIP - Semaphore (VIP)", "type": "http", "url": "http://140.44.4.58:100", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Spaceport VIP - Jenkins (VIP)", "type": "http", "url": "http://140.44.4.59:101", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Spaceport VIP - Homepage (VIP)", "type": "http", "url": "http://140.44.4.60:8040", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Spaceport VIP - Gitea (VIP)", "type": "http", "url": "http://140.44.4.62:90", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Spaceport Node - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.41:80", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Home Assistant Core (IP)", "type": "http", "url": "http://14.4.4.20:8123", "interval": 30, "ignore_tls": 0},
|
|
{"name": "AI Hailo Engine (IP)", "type": "http", "url": "http://140.44.4.7:8000/docs", "interval": 30, "ignore_tls": 0},
|
|
{"name": "AI Open WebUI (IP)", "type": "http", "url": "http://140.44.4.8:8080", "interval": 30, "ignore_tls": 0},
|
|
{"name": "Harvester HCI Console (VIP)", "type": "http", "url": "https://140.44.4.90:443", "interval": 30, "ignore_tls": 1},
|
|
{"name": "Synology NAS 1 DSM (IP)", "type": "http", "url": "https://14.10.10.71:5001", "interval": 30, "ignore_tls": 1},
|
|
{"name": "SnarfNAS DSM (IP)", "type": "http", "url": "https://14.10.10.72:5001", "interval": 30, "ignore_tls": 1},
|
|
{"name": "PeaNUT UPS Daemon (VIP)", "type": "http", "url": "http://140.44.4.70:8050/api/v1/devices", "interval": 30, "ignore_tls": 0},
|
|
]
|
|
|
|
# Fetch existing monitors to prevent duplicates
|
|
existing_out = query_db("SELECT id, name FROM monitor;")
|
|
existing_names = set()
|
|
for line in existing_out.splitlines()[1:]:
|
|
parts = line.split("\\t")
|
|
if len(parts) >= 2:
|
|
existing_names.add(parts[1].strip())
|
|
|
|
added_count = 0
|
|
for m in monitors:
|
|
name = m["name"]
|
|
if name in existing_names:
|
|
print(f"Skipping existing monitor: {name}")
|
|
continue
|
|
|
|
m_type = m["type"]
|
|
interval = m.get("interval", 60)
|
|
ignore_tls = m.get("ignore_tls", 0)
|
|
status_codes = m.get("status_codes", '["200-299"]')
|
|
|
|
if m_type == "http":
|
|
url = m["url"]
|
|
sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, url, ignore_tls, accepted_statuscodes_json)
|
|
VALUES ('{name}', 'http', 1, 1, {interval}, '{url}', {ignore_tls}, '{status_codes}');"""
|
|
elif m_type == "port":
|
|
host = m["hostname"]
|
|
p = m["port"]
|
|
sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, hostname, port)
|
|
VALUES ('{name}', 'port', 1, 1, {interval}, '{host}', {p});"""
|
|
|
|
query_db(sql)
|
|
added_count += 1
|
|
print(f"Added monitor: {name}")
|
|
|
|
# Link all monitors to Gotify notification
|
|
all_monitors_out = query_db("SELECT id FROM monitor;")
|
|
for line in all_monitors_out.splitlines()[1:]:
|
|
line = line.strip()
|
|
if line.isdigit():
|
|
m_id = line
|
|
check_link = query_db(f"SELECT id FROM monitor_notification WHERE monitor_id = {m_id} AND notification_id = {notif_id};")
|
|
if "id" not in check_link or not check_link.split():
|
|
query_db(f"INSERT INTO monitor_notification (monitor_id, notification_id) VALUES ({m_id}, {notif_id});")
|
|
|
|
total_count = query_db("SELECT count(*) FROM monitor;")
|
|
print(f"\\nRecovery Complete! Added {added_count} monitors. Total active monitors in DB: {total_count.strip()}")
|
|
'''
|
|
|
|
stdin, _, _ = c.exec_command("cat > /tmp/recover_kuma.py")
|
|
stdin.write(setup_script)
|
|
stdin.close()
|
|
|
|
run_cmd(f"kubectl cp /tmp/recover_kuma.py monitoring/{pod}:/tmp/recover_kuma.py")
|
|
run_cmd(f"kubectl exec -n monitoring {pod} -- python3 /tmp/recover_kuma.py")
|
|
|
|
print("\n=== Restarting Uptime Kuma Deployment ===")
|
|
run_cmd("kubectl rollout restart deployment/uptime-kuma-2-1756668961 -n monitoring")
|
|
run_cmd("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=90s")
|
|
|
|
c.close()
|
|
print("\n=== Recovery Successfully Finished ===")
|
|
|