88 lines
3.6 KiB
Python
88 lines
3.6 KiB
Python
import paramiko
|
|
import json
|
|
import urllib.request
|
|
import ssl
|
|
|
|
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
|
|
|
def run_ssh(host, user, password, cmd, sudo=False):
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10)
|
|
full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd
|
|
stdin, stdout, stderr = client.exec_command(full_cmd)
|
|
out = stdout.read().decode('utf-8', errors='ignore').strip()
|
|
err = stderr.read().decode('utf-8', errors='ignore').strip()
|
|
client.close()
|
|
return out, err
|
|
except Exception as e:
|
|
return "", str(e)
|
|
|
|
print("=== 1. CHECK LONGHORN VOLUME FAULTS & SALVAGE ===")
|
|
k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; "
|
|
cmd = k8s_env + "kubectl get volume.longhorn.io -n longhorn-system -o json"
|
|
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", cmd, sudo=True)
|
|
try:
|
|
vols = json.loads(out)
|
|
for v in vols.get("items", []):
|
|
name = v["metadata"]["name"]
|
|
status = v.get("status", {})
|
|
spec = v.get("spec", {})
|
|
print(f"Volume: {name}")
|
|
print(f" State: {status.get('state')}, Robustness: {status.get('robustness')}")
|
|
print(f" SalvageRequested: {status.get('salvageRequested')}, Conditions: {status.get('conditions')}")
|
|
except Exception as e:
|
|
print("Error parsing volumes json:", e)
|
|
|
|
# Check Longhorn manager pod logs
|
|
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl logs -n longhorn-system -l app=longhorn-manager --tail=30", sudo=True)
|
|
print("\nLonghorn manager recent logs:")
|
|
print(out[:1000])
|
|
|
|
print("\n=== 2. HEIMDALL MOUNT & SQLITE BACKUP ===")
|
|
# Inspect Heimdall service spec on 140.44.4.71
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'", sudo=True)
|
|
print("Heimdall Service Mounts:", out)
|
|
|
|
# Check running container details
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker inspect $(docker ps -q -f name=heimdall) --format '{{json .Mounts}}'", sudo=True)
|
|
print("Heimdall Container Mounts:", out)
|
|
|
|
# Check SQLite DB using python on the host
|
|
py_check_db = """
|
|
python3 -c "
|
|
import sqlite3
|
|
import shutil
|
|
|
|
src = '/var/lib/docker/volumes/heimdall_heimdall_config/_data/www/app.sqlite'
|
|
# Backup first!
|
|
shutil.copyfile(src, src + '.bak_' + str(int(__import__('time').time())))
|
|
print('Backup created!')
|
|
|
|
conn = sqlite3.connect(src)
|
|
c = conn.cursor()
|
|
print('Users:')
|
|
for row in c.execute('SELECT id, username, email, public_front FROM users').fetchall():
|
|
print(' ', row)
|
|
print('Items count:', c.execute('SELECT count(*) FROM items').fetchone()[0])
|
|
print('Pinned items count:', c.execute('SELECT count(*) FROM items WHERE pinned=1').fetchone()[0])
|
|
print('Items list:')
|
|
for row in c.execute('SELECT id, title, pinned, user_id, apporder FROM items').fetchall():
|
|
print(' ', row)
|
|
"
|
|
"""
|
|
out, err = run_ssh("140.44.4.71", "snarf", "Katzir476!", py_check_db, sudo=True)
|
|
print("Heimdall DB check output:")
|
|
print(out if out else err)
|
|
|
|
# Check heimdall container logs from the restart 5 hours ago
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker logs $(docker ps -q -f name=heimdall) --tail=60", sudo=True)
|
|
print("\nHeimdall Container logs:")
|
|
print(out)
|
|
|
|
print("\n=== 3. DOCKER CONTAINERS ON 140.44.4.71 (Proxy, Kuma, etc.) ===")
|
|
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'", sudo=True)
|
|
print(out)
|
|
|