feat: add various infrastructure check and automation scripts

This commit is contained in:
2026-11-15 17:17:41 -05:00
parent 336b7759d0
commit b301aa3ece
232 changed files with 8274 additions and 298 deletions

View File

@@ -0,0 +1,87 @@
import paramiko
import json
import urllib.request
import ssl
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
def run_ssh(host, user, password, cmd, sudo=False):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect(host, username=user, key_filename=SSH_KEY, password=password, timeout=10)
full_cmd = f"echo {password} | sudo -S sh -c '{cmd}'" if sudo else cmd
stdin, stdout, stderr = client.exec_command(full_cmd)
out = stdout.read().decode('utf-8', errors='ignore').strip()
err = stderr.read().decode('utf-8', errors='ignore').strip()
client.close()
return out, err
except Exception as e:
return "", str(e)
print("=== 1. CHECK LONGHORN VOLUME FAULTS & SALVAGE ===")
k8s_env = "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; "
cmd = k8s_env + "kubectl get volume.longhorn.io -n longhorn-system -o json"
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", cmd, sudo=True)
try:
vols = json.loads(out)
for v in vols.get("items", []):
name = v["metadata"]["name"]
status = v.get("status", {})
spec = v.get("spec", {})
print(f"Volume: {name}")
print(f" State: {status.get('state')}, Robustness: {status.get('robustness')}")
print(f" SalvageRequested: {status.get('salvageRequested')}, Conditions: {status.get('conditions')}")
except Exception as e:
print("Error parsing volumes json:", e)
# Check Longhorn manager pod logs
out, _ = run_ssh("140.44.4.90", "rancher", "Katzir476!", k8s_env + "kubectl logs -n longhorn-system -l app=longhorn-manager --tail=30", sudo=True)
print("\nLonghorn manager recent logs:")
print(out[:1000])
print("\n=== 2. HEIMDALL MOUNT & SQLITE BACKUP ===")
# Inspect Heimdall service spec on 140.44.4.71
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'", sudo=True)
print("Heimdall Service Mounts:", out)
# Check running container details
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker inspect $(docker ps -q -f name=heimdall) --format '{{json .Mounts}}'", sudo=True)
print("Heimdall Container Mounts:", out)
# Check SQLite DB using python on the host
py_check_db = """
python3 -c "
import sqlite3
import shutil
src = '/var/lib/docker/volumes/heimdall_heimdall_config/_data/www/app.sqlite'
# Backup first!
shutil.copyfile(src, src + '.bak_' + str(int(__import__('time').time())))
print('Backup created!')
conn = sqlite3.connect(src)
c = conn.cursor()
print('Users:')
for row in c.execute('SELECT id, username, email, public_front FROM users').fetchall():
print(' ', row)
print('Items count:', c.execute('SELECT count(*) FROM items').fetchone()[0])
print('Pinned items count:', c.execute('SELECT count(*) FROM items WHERE pinned=1').fetchone()[0])
print('Items list:')
for row in c.execute('SELECT id, title, pinned, user_id, apporder FROM items').fetchall():
print(' ', row)
"
"""
out, err = run_ssh("140.44.4.71", "snarf", "Katzir476!", py_check_db, sudo=True)
print("Heimdall DB check output:")
print(out if out else err)
# Check heimdall container logs from the restart 5 hours ago
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker logs $(docker ps -q -f name=heimdall) --tail=60", sudo=True)
print("\nHeimdall Container logs:")
print(out)
print("\n=== 3. DOCKER CONTAINERS ON 140.44.4.71 (Proxy, Kuma, etc.) ===")
out, _ = run_ssh("140.44.4.71", "snarf", "Katzir476!", "docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'", sudo=True)
print(out)