156 lines
6.8 KiB
Python
156 lines
6.8 KiB
Python
import paramiko
|
|
import json
|
|
import urllib.request
|
|
import urllib.parse
|
|
import ssl
|
|
|
|
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
|
|
|
def run_ssh(host, user, password, cmd, sudo=False):
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10)
|
|
full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd
|
|
stdin, stdout, stderr = client.exec_command(full_cmd)
|
|
out = stdout.read().decode('utf-8', errors='ignore').strip()
|
|
err = stderr.read().decode('utf-8', errors='ignore').strip()
|
|
client.close()
|
|
return out, err
|
|
except Exception as e:
|
|
return "", str(e)
|
|
|
|
print("=== 1. HARVESTER / LONGHORN VOLUME FAULT INSPECTION ===")
|
|
k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; "
|
|
# Check pods in ai-stack
|
|
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get pods -n ai-stack -o wide", sudo=True)
|
|
print("Pods in ai-stack:")
|
|
print(out)
|
|
|
|
# Check Longhorn volume details for one of the faulted volumes
|
|
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get volume.longhorn.io pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495 -n longhorn-system -o yaml | grep -E 'robustness|state|salvage|currentReplicas|conditions'", sudo=True)
|
|
print("Volume details (sample):")
|
|
print(out)
|
|
|
|
# Check replicas for the volumes
|
|
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl get replicas.longhorn.io -n longhorn-system", sudo=True)
|
|
print("Replicas summary:")
|
|
for line in out.splitlines()[:15]:
|
|
print(line)
|
|
|
|
print("\n=== 2. HEIMDALL DATABASE & TILES INSPECTION ===")
|
|
# Find container ID on 140.44.4.71
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --filter name=heimdall -q", sudo=True)
|
|
cid = out.splitlines()[0] if out else ""
|
|
print(f"Heimdall Container ID: {cid}")
|
|
if cid:
|
|
# Inspect mounts
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker inspect {cid} --format '{{{{json .Mounts}}}}'", sudo=True)
|
|
print("Mounts:", out)
|
|
# Check sqlite file location inside container
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} ls -la /config/www", sudo=True)
|
|
print("Files in /config/www:")
|
|
print(out)
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} sqlite3 /config/www/app.sqlite 'SELECT id, title, pinned, user_id FROM items;' 2>/dev/null", sudo=True)
|
|
print("Items in app.sqlite:")
|
|
print(out)
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec {cid} sqlite3 /config/www/app.sqlite 'SELECT id, username, email FROM users;' 2>/dev/null", sudo=True)
|
|
print("Users in app.sqlite:")
|
|
print(out)
|
|
# Check host mount path files
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "ls -la /var/lib/docker/volumes/heimdall_heimdall_config/_data", sudo=True)
|
|
print("Host volume data dir:")
|
|
print(out)
|
|
|
|
print("\n=== 3. AUTHENTIK & NPM REVERSE PROXY FOR HEIMDALL ===")
|
|
# Check NPM config for heimdall
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker exec $(docker ps -q -f name=npm) grep -rnw '/data/nginx' -e 'heimdall' 2>/dev/null || docker exec $(docker ps -q -f name=proxy) grep -rn 'heimdall' /etc/nginx/conf.d/ 2>/dev/null", sudo=True)
|
|
print("NPM config search:")
|
|
print(out)
|
|
|
|
# Check NPM proxy hosts
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker exec $(docker ps -q -f name=npm) find /data/nginx/proxy_host/ -type f -exec grep -l 'heimdall' {} + 2>/dev/null", sudo=True)
|
|
print("NPM proxy host file:")
|
|
print(out)
|
|
if out.strip():
|
|
conf_file = out.strip().splitlines()[0]
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", f"docker exec $(docker ps -q -f name=npm) cat {conf_file}", sudo=True)
|
|
print(f"NPM proxy host content ({conf_file}):")
|
|
print(out)
|
|
|
|
print("\n=== 4. ZABBIX API - SPOCKDOCK MONITORING & ALERT THRESHOLDS ===")
|
|
ctx = ssl.create_default_context()
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
|
|
def zabbix_api(method, params, auth=None):
|
|
payload = {
|
|
"jsonrpc": "2.0",
|
|
"method": method,
|
|
"params": params,
|
|
"id": 1
|
|
}
|
|
if auth:
|
|
payload["auth"] = auth
|
|
req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php",
|
|
data=json.dumps(payload).encode('utf-8'),
|
|
headers={"Content-Type": "application/json-rpc"})
|
|
try:
|
|
resp = urllib.request.urlopen(req, context=ctx, timeout=10)
|
|
return json.loads(resp.read().decode('utf-8'))
|
|
except Exception as e:
|
|
return {"error": str(e)}
|
|
|
|
auth_res = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"})
|
|
token = auth_res.get("result")
|
|
print("Zabbix Auth token obtained:", bool(token))
|
|
|
|
if token:
|
|
# 4a. Check Spacedock hosts
|
|
hosts = zabbix_api("host.get", {
|
|
"output": ["hostid", "host", "name", "status"],
|
|
"search": {"name": "spacedock"},
|
|
"searchWildcardsEnabled": True
|
|
}, auth=token)
|
|
print("\nZabbix Spacedock hosts:")
|
|
print(json.dumps(hosts.get("result", []), indent=2))
|
|
|
|
# Also search by IP
|
|
hosts_ip = zabbix_api("host.get", {
|
|
"output": ["hostid", "host", "name", "status"],
|
|
"filter": {"ip": ["140.44.4.90", "140.44.4.81", "140.44.4.82", "140.44.4.83", "140.44.4.84"]},
|
|
"selectInterfaces": ["ip", "type", "available"]
|
|
}, auth=token)
|
|
print("\nZabbix hosts by Harvester IPs:")
|
|
print(json.dumps(hosts_ip.get("result", []), indent=2))
|
|
|
|
# Check iDRAC hosts
|
|
hosts_idrac = zabbix_api("host.get", {
|
|
"output": ["hostid", "host", "name", "status"],
|
|
"search": {"name": "idrac"},
|
|
"searchWildcardsEnabled": True,
|
|
"selectInterfaces": ["ip", "type", "available"]
|
|
}, auth=token)
|
|
print("\nZabbix iDRAC hosts:")
|
|
print(json.dumps(hosts_idrac.get("result", []), indent=2))
|
|
|
|
# 4b. Check CPU and Memory triggers firing or high noise
|
|
triggers = zabbix_api("trigger.get", {
|
|
"output": ["triggerid", "description", "expression", "priority", "status", "value"],
|
|
"search": {"description": "CPU"},
|
|
"limit": 10
|
|
}, auth=token)
|
|
print("\nSample CPU Triggers in Zabbix:")
|
|
for t in triggers.get("result", []):
|
|
print(f"ID: {t['triggerid']}, Desc: {t['description']}, Expression: {t['expression']}, Priority: {t['priority']}")
|
|
|
|
triggers_mem = zabbix_api("trigger.get", {
|
|
"output": ["triggerid", "description", "expression", "priority", "status", "value"],
|
|
"search": {"description": "Memory"},
|
|
"limit": 10
|
|
}, auth=token)
|
|
print("\nSample Memory Triggers in Zabbix:")
|
|
for t in triggers_mem.get("result", []):
|
|
print(f"ID: {t['triggerid']}, Desc: {t['description']}, Expression: {t['expression']}, Priority: {t['priority']}")
|
|
|