import paramiko import json import urllib.request import ssl SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" def run_ssh(host, user, password, cmd, sudo=False): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10) full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd stdin, stdout, stderr = client.exec_command(full_cmd) out = stdout.read().decode('utf-8', errors='ignore').strip() err = stderr.read().decode('utf-8', errors='ignore').strip() client.close() return out, err except Exception as e: return "", str(e) print("=== 1. CHECK LONGHORN VOLUME FAULTS & SALVAGE ===") k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; " cmd = k8s_env + "kubectl get volume.longhorn.io -n longhorn-system -o json" out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", cmd, sudo=True) try: vols = json.loads(out) for v in vols.get("items", []): name = v["metadata"]["name"] status = v.get("status", {}) spec = v.get("spec", {}) print(f"Volume: {name}") print(f" State: {status.get('state')}, Robustness: {status.get('robustness')}") print(f" SalvageRequested: {status.get('salvageRequested')}, Conditions: {status.get('conditions')}") except Exception as e: print("Error parsing volumes json:", e) # Check Longhorn manager pod logs out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl logs -n longhorn-system -l app=longhorn-manager --tail=30", sudo=True) print("\nLonghorn manager recent logs:") print(out[:1000]) print("\n=== 2. HEIMDALL MOUNT & SQLITE BACKUP ===") # Inspect Heimdall service spec on 140.44.4.71 out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'", sudo=True) print("Heimdall Service Mounts:", out) # Check running container details out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker inspect $(docker ps -q -f name=heimdall) --format '{{json .Mounts}}'", sudo=True) print("Heimdall Container Mounts:", out) # Check SQLite DB using python on the host py_check_db = """ python3 -c " import sqlite3 import shutil src = '/var/lib/docker/volumes/heimdall_heimdall_config/_data/www/app.sqlite' # Backup first! shutil.copyfile(src, src + '.bak_' + str(int(__import__('time').time()))) print('Backup created!') conn = sqlite3.connect(src) c = conn.cursor() print('Users:') for row in c.execute('SELECT id, username, email, public_front FROM users').fetchall(): print(' ', row) print('Items count:', c.execute('SELECT count(*) FROM items').fetchone()[0]) print('Pinned items count:', c.execute('SELECT count(*) FROM items WHERE pinned=1').fetchone()[0]) print('Items list:') for row in c.execute('SELECT id, title, pinned, user_id, apporder FROM items').fetchall(): print(' ', row) " """ out, err = run_ssh("140.44.4.71", "snarf", "Katzir476!", py_check_db, sudo=True) print("Heimdall DB check output:") print(out if out else err) # Check heimdall container logs from the restart 5 hours ago out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker logs $(docker ps -q -f name=heimdall) --tail=60", sudo=True) print("\nHeimdall Container logs:") print(out) print("\n=== 3. DOCKER CONTAINERS ON 140.44.4.71 (Proxy, Kuma, etc.) ===") out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'", sudo=True) print(out)