feat: add various infrastructure check and automation scripts
This commit is contained in:
87
scripts/deep_investigate_p1.py
Normal file
87
scripts/deep_investigate_p1.py
Normal file
@@ -0,0 +1,87 @@
|
||||
import paramiko
|
||||
import json
|
||||
import urllib.request
|
||||
import ssl
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
|
||||
def run_ssh(host, user, password, cmd, sudo=False):
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10)
|
||||
full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd
|
||||
stdin, stdout, stderr = client.exec_command(full_cmd)
|
||||
out = stdout.read().decode('utf-8', errors='ignore').strip()
|
||||
err = stderr.read().decode('utf-8', errors='ignore').strip()
|
||||
client.close()
|
||||
return out, err
|
||||
except Exception as e:
|
||||
return "", str(e)
|
||||
|
||||
print("=== 1. CHECK LONGHORN VOLUME FAULTS & SALVAGE ===")
|
||||
k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; "
|
||||
cmd = k8s_env + "kubectl get volume.longhorn.io -n longhorn-system -o json"
|
||||
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", cmd, sudo=True)
|
||||
try:
|
||||
vols = json.loads(out)
|
||||
for v in vols.get("items", []):
|
||||
name = v["metadata"]["name"]
|
||||
status = v.get("status", {})
|
||||
spec = v.get("spec", {})
|
||||
print(f"Volume: {name}")
|
||||
print(f" State: {status.get('state')}, Robustness: {status.get('robustness')}")
|
||||
print(f" SalvageRequested: {status.get('salvageRequested')}, Conditions: {status.get('conditions')}")
|
||||
except Exception as e:
|
||||
print("Error parsing volumes json:", e)
|
||||
|
||||
# Check Longhorn manager pod logs
|
||||
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl logs -n longhorn-system -l app=longhorn-manager --tail=30", sudo=True)
|
||||
print("\nLonghorn manager recent logs:")
|
||||
print(out[:1000])
|
||||
|
||||
print("\n=== 2. HEIMDALL MOUNT & SQLITE BACKUP ===")
|
||||
# Inspect Heimdall service spec on 140.44.4.71
|
||||
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'", sudo=True)
|
||||
print("Heimdall Service Mounts:", out)
|
||||
|
||||
# Check running container details
|
||||
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker inspect $(docker ps -q -f name=heimdall) --format '{{json .Mounts}}'", sudo=True)
|
||||
print("Heimdall Container Mounts:", out)
|
||||
|
||||
# Check SQLite DB using python on the host
|
||||
py_check_db = """
|
||||
python3 -c "
|
||||
import sqlite3
|
||||
import shutil
|
||||
|
||||
src = '/var/lib/docker/volumes/heimdall_heimdall_config/_data/www/app.sqlite'
|
||||
# Backup first!
|
||||
shutil.copyfile(src, src + '.bak_' + str(int(__import__('time').time())))
|
||||
print('Backup created!')
|
||||
|
||||
conn = sqlite3.connect(src)
|
||||
c = conn.cursor()
|
||||
print('Users:')
|
||||
for row in c.execute('SELECT id, username, email, public_front FROM users').fetchall():
|
||||
print(' ', row)
|
||||
print('Items count:', c.execute('SELECT count(*) FROM items').fetchone()[0])
|
||||
print('Pinned items count:', c.execute('SELECT count(*) FROM items WHERE pinned=1').fetchone()[0])
|
||||
print('Items list:')
|
||||
for row in c.execute('SELECT id, title, pinned, user_id, apporder FROM items').fetchall():
|
||||
print(' ', row)
|
||||
"
|
||||
"""
|
||||
out, err = run_ssh("140.44.4.71", "snarf", "Katzir476!", py_check_db, sudo=True)
|
||||
print("Heimdall DB check output:")
|
||||
print(out if out else err)
|
||||
|
||||
# Check heimdall container logs from the restart 5 hours ago
|
||||
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker logs $(docker ps -q -f name=heimdall) --tail=60", sudo=True)
|
||||
print("\nHeimdall Container logs:")
|
||||
print(out)
|
||||
|
||||
print("\n=== 3. DOCKER CONTAINERS ON 140.44.4.71 (Proxy, Kuma, etc.) ===")
|
||||
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'", sudo=True)
|
||||
print(out)
|
||||
|
||||
Reference in New Issue
Block a user