import paramiko import json import urllib.request import urllib.parse import ssl SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" def run_ssh(host, user, password, cmd, sudo=False): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10) full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd stdin, stdout, stderr = client.exec_command(full_cmd) out = stdout.read().decode('utf-8', errors='ignore').strip() err = stderr.read().decode('utf-8', errors='ignore').strip() client.close() return out, err except Exception as e: return "", str(e) print("=== 1. HARVESTER / LONGHORN VOLUME FAULT INSPECTION ===") k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; " # Check pods in ai-stack out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get pods -n ai-stack -o wide", sudo=True) print("Pods in ai-stack:") print(out) # Check Longhorn volume details for one of the faulted volumes out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get volume.longhorn.io pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495 -n longhorn-system -o yaml | grep -E 'robustness|state|salvage|currentReplicas|conditions'", sudo=True) print("Volume details (sample):") print(out) # Check replicas for the volumes out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get replicas.longhorn.io -n longhorn-system", sudo=True) print("Replicas summary:") for line in out.splitlines()[:15]: print(line) print("\n=== 2. HEIMDALL DATABASE & TILES INSPECTION ===") # Find container ID on 140.44.4.71 out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --filter name=heimdall -q", sudo=True) cid = out.splitlines()[0] if out else "" print(f"Heimdall Container ID: {cid}") if cid: # Inspect mounts out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker inspect {cid} --format '{{{{json .Mounts}}}}'", sudo=True) print("Mounts:", out) # Check sqlite file location inside container out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} ls -la /config/www", sudo=True) print("Files in /config/www:") print(out) out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} sqlite3 /config/www/app.sqlite 'SELECT id, title, pinned, user_id FROM items;' 2>/dev/null", sudo=True) print("Items in app.sqlite:") print(out) out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} sqlite3 /config/www/app.sqlite 'SELECT id, username, email FROM users;' 2>/dev/null", sudo=True) print("Users in app.sqlite:") print(out) # Check host mount path files out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "ls -la /var/lib/docker/volumes/heimdall_heimdall_config/_data", sudo=True) print("Host volume data dir:") print(out) print("\n=== 3. AUTHENTIK & NPM REVERSE PROXY FOR HEIMDALL ===") # Check NPM config for heimdall out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker exec $(docker ps -q -f name=npm) grep -rnw '/data/nginx' -e 'heimdall' 2>/dev/null || docker exec $(docker ps -q -f name=proxy) grep -rn 'heimdall' /etc/nginx/conf.d/ 2>/dev/null", sudo=True) print("NPM config search:") print(out) # Check NPM proxy hosts out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker exec $(docker ps -q -f name=npm) find /data/nginx/proxy_host/ -type f -exec grep -l 'heimdall' {} + 2>/dev/null", sudo=True) print("NPM proxy host file:") print(out) if out.strip(): conf_file = out.strip().splitlines()[0] out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec $(docker ps -q -f name=npm) cat {conf_file}", sudo=True) print(f"NPM proxy host content ({conf_file}):") print(out) print("\n=== 4. ZABBIX API - SPOCKDOCK MONITORING & ALERT THRESHOLDS ===") ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE def zabbix_api(method, params, auth=None): payload = { "jsonrpc": "2.0", "method": method, "params": params, "id": 1 } if auth: payload["auth"] = auth req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php", data=json.dumps(payload).encode('utf-8'), headers={"Content-Type": "application/json-rpc"}) try: resp = urllib.request.urlopen(req, context=ctx, timeout=10) return json.loads(resp.read().decode('utf-8')) except Exception as e: return {"error": str(e)} auth_res = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"}) token = auth_res.get("result") print("Zabbix Auth token obtained:", bool(token)) if token: # 4a. Check Spacedock hosts hosts = zabbix_api("host.get", { "output": ["hostid", "host", "name", "status"], "search": {"name": "spacedock"}, "searchWildcardsEnabled": True }, auth=token) print("\nZabbix Spacedock hosts:") print(json.dumps(hosts.get("result", []), indent=2)) # Also search by IP hosts_ip = zabbix_api("host.get", { "output": ["hostid", "host", "name", "status"], "filter": {"ip": ["140.44.4.90", "140.44.4.81", "140.44.4.82", "140.44.4.83", "140.44.4.84"]}, "selectInterfaces": ["ip", "type", "available"] }, auth=token) print("\nZabbix hosts by Harvester IPs:") print(json.dumps(hosts_ip.get("result", []), indent=2)) # Check iDRAC hosts hosts_idrac = zabbix_api("host.get", { "output": ["hostid", "host", "name", "status"], "search": {"name": "idrac"}, "searchWildcardsEnabled": True, "selectInterfaces": ["ip", "type", "available"] }, auth=token) print("\nZabbix iDRAC hosts:") print(json.dumps(hosts_idrac.get("result", []), indent=2)) # 4b. Check CPU and Memory triggers firing or high noise triggers = zabbix_api("trigger.get", { "output": ["triggerid", "description", "expression", "priority", "status", "value"], "search": {"description": "CPU"}, "limit": 10 }, auth=token) print("\nSample CPU Triggers in Zabbix:") for t in triggers.get("result", []): print(f"ID: {t['triggerid']}, Desc: {t['description']}, Expression: {t['expression']}, Priority: {t['priority']}") triggers_mem = zabbix_api("trigger.get", { "output": ["triggerid", "description", "expression", "priority", "status", "value"], "search": {"description": "Memory"}, "limit": 10 }, auth=token) print("\nSample Memory Triggers in Zabbix:") for t in triggers_mem.get("result", []): print(f"ID: {t['triggerid']}, Desc: {t['description']}, Expression: {t['expression']}, Priority: {t['priority']}")