feat: add various infrastructure check and automation scripts

This commit is contained in:
2026-11-15 17:17:41 -05:00
parent 336b7759d0
commit b301aa3ece
232 changed files with 8274 additions and 298 deletions

155
scripts/deep_investigate.py Normal file
View File

@@ -0,0 +1,155 @@
import paramiko
import json
import urllib.request
import urllib.parse
import ssl
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
def run_ssh(host, user, password, cmd, sudo=False):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10)
full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd
stdin, stdout, stderr = client.exec_command(full_cmd)
out = stdout.read().decode('utf-8', errors='ignore').strip()
err = stderr.read().decode('utf-8', errors='ignore').strip()
client.close()
return out, err
except Exception as e:
return "", str(e)
print("=== 1. HARVESTER / LONGHORN VOLUME FAULT INSPECTION ===")
k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; "
# Check pods in ai-stack
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get pods -n ai-stack -o wide", sudo=True)
print("Pods in ai-stack:")
print(out)
# Check Longhorn volume details for one of the faulted volumes
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get volume.longhorn.io pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495 -n longhorn-system -o yaml | grep -E 'robustness|state|salvage|currentReplicas|conditions'", sudo=True)
print("Volume details (sample):")
print(out)
# Check replicas for the volumes
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get replicas.longhorn.io -n longhorn-system", sudo=True)
print("Replicas summary:")
for line in out.splitlines()[:15]:
print(line)
print("\n=== 2. HEIMDALL DATABASE & TILES INSPECTION ===")
# Find container ID on 140.44.4.71
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --filter name=heimdall -q", sudo=True)
cid = out.splitlines()[0] if out else ""
print(f"Heimdall Container ID: {cid}")
if cid:
# Inspect mounts
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker inspect {cid} --format '{{{{json .Mounts}}}}'", sudo=True)
print("Mounts:", out)
# Check sqlite file location inside container
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} ls -la /config/www", sudo=True)
print("Files in /config/www:")
print(out)
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} sqlite3 /config/www/app.sqlite 'SELECT id, title, pinned, user_id FROM items;' 2>/dev/null", sudo=True)
print("Items in app.sqlite:")
print(out)
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} sqlite3 /config/www/app.sqlite 'SELECT id, username, email FROM users;' 2>/dev/null", sudo=True)
print("Users in app.sqlite:")
print(out)
# Check host mount path files
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "ls -la /var/lib/docker/volumes/heimdall_heimdall_config/_data", sudo=True)
print("Host volume data dir:")
print(out)
print("\n=== 3. AUTHENTIK & NPM REVERSE PROXY FOR HEIMDALL ===")
# Check NPM config for heimdall
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker exec $(docker ps -q -f name=npm) grep -rnw '/data/nginx' -e 'heimdall' 2>/dev/null || docker exec $(docker ps -q -f name=proxy) grep -rn 'heimdall' /etc/nginx/conf.d/ 2>/dev/null", sudo=True)
print("NPM config search:")
print(out)
# Check NPM proxy hosts
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker exec $(docker ps -q -f name=npm) find /data/nginx/proxy_host/ -type f -exec grep -l 'heimdall' {} + 2>/dev/null", sudo=True)
print("NPM proxy host file:")
print(out)
if out.strip():
conf_file = out.strip().splitlines()[0]
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec $(docker ps -q -f name=npm) cat {conf_file}", sudo=True)
print(f"NPM proxy host content ({conf_file}):")
print(out)
print("\n=== 4. ZABBIX API - SPOCKDOCK MONITORING & ALERT THRESHOLDS ===")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def zabbix_api(method, params, auth=None):
payload = {
"jsonrpc": "2.0",
"method": method,
"params": params,
"id": 1
}
if auth:
payload["auth"] = auth
req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php",
data=json.dumps(payload).encode('utf-8'),
headers={"Content-Type": "application/json-rpc"})
try:
resp = urllib.request.urlopen(req, context=ctx, timeout=10)
return json.loads(resp.read().decode('utf-8'))
except Exception as e:
return {"error": str(e)}
auth_res = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"})
token = auth_res.get("result")
print("Zabbix Auth token obtained:", bool(token))
if token:
# 4a. Check Spacedock hosts
hosts = zabbix_api("host.get", {
"output": ["hostid", "host", "name", "status"],
"search": {"name": "spacedock"},
"searchWildcardsEnabled": True
}, auth=token)
print("\nZabbix Spacedock hosts:")
print(json.dumps(hosts.get("result", []), indent=2))
# Also search by IP
hosts_ip = zabbix_api("host.get", {
"output": ["hostid", "host", "name", "status"],
"filter": {"ip": ["140.44.4.90", "140.44.4.81", "140.44.4.82", "140.44.4.83", "140.44.4.84"]},
"selectInterfaces": ["ip", "type", "available"]
}, auth=token)
print("\nZabbix hosts by Harvester IPs:")
print(json.dumps(hosts_ip.get("result", []), indent=2))
# Check iDRAC hosts
hosts_idrac = zabbix_api("host.get", {
"output": ["hostid", "host", "name", "status"],
"search": {"name": "idrac"},
"searchWildcardsEnabled": True,
"selectInterfaces": ["ip", "type", "available"]
}, auth=token)
print("\nZabbix iDRAC hosts:")
print(json.dumps(hosts_idrac.get("result", []), indent=2))
# 4b. Check CPU and Memory triggers firing or high noise
triggers = zabbix_api("trigger.get", {
"output": ["triggerid", "description", "expression", "priority", "status", "value"],
"search": {"description": "CPU"},
"limit": 10
}, auth=token)
print("\nSample CPU Triggers in Zabbix:")
for t in triggers.get("result", []):
print(f"ID: {t['triggerid']}, Desc: {t['description']}, Expression: {t['expression']}, Priority: {t['priority']}")
triggers_mem = zabbix_api("trigger.get", {
"output": ["triggerid", "description", "expression", "priority", "status", "value"],
"search": {"description": "Memory"},
"limit": 10
}, auth=token)
print("\nSample Memory Triggers in Zabbix:")
for t in triggers_mem.get("result", []):
print(f"ID: {t['triggerid']}, Desc: {t['description']}, Expression: {t['expression']}, Priority: {t['priority']}")