Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add scripts to audit and troubleshoot spaceport kernels and cluster nodes

Browse Source
This commit is contained in:
Snarf
2026-11-05 07:28:22 -05:00
parent b974980d08
commit 14fac7151f
19 changed files with 327 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
scripts/audit_spaceport_kernels.py Normal file
View File

@@ -0,0 +1,39 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
spaceport_nodes = [
('sninfr0lnxctl01', '140.44.4.41'),
('sninfr0lnxctl02', '140.44.4.42'),
('sninfr0lnxctl03', '140.44.4.43'),
('sninfr0lnxctl04', '140.44.4.44'),
('sninfr0lnxwrk01', '140.44.4.51'),
('sninfr0lnxwrk02', '140.44.4.52'),
('sninfr0lnxwrk03', '140.44.4.53'),
('sninfr0lnxwrk04', '140.44.4.54'),
('sninfr0lnxwrk05', '140.44.4.55'),
('sninfr0lnxwrk06', '140.44.4.56'),
]
print(f"{'NODE':<18} {'IP':<15} {'RUNNING KERNEL':<22} {'LATEST MODULES':<25} {'VETH LOADED':<12} {'UPTIME'}")
print("="*105)
for name, ip in spaceport_nodes:
try:
client.connect(ip, username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = client.exec_command("uname -r; ls -1 /lib/modules | sort -V | tail -n 1; lsmod | grep -q veth && echo YES || echo NO; uptime -p")
lines = [l.strip() for l in stdout.read().decode('utf-8').strip().split('\n') if l.strip()]
running_k = lines[0] if len(lines) > 0 else "?"
latest_mod = lines[1] if len(lines) > 1 else "?"
veth = lines[2] if len(lines) > 2 else "?"
uptime_str = lines[3] if len(lines) > 3 else "?"
print(f"{name:<18} {ip:<15} {running_k:<22} {latest_mod:<25} {veth:<12} {uptime_str}")
client.close()
except Exception as e:
print(f"{name:<18} {ip:<15} ERROR: {e}")

13
scripts/check_blkid.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("blkid")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_boot_ctl03.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("ls -la /boot; ls -la /boot/firmware 2>/dev/null | head -n 20")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_cattle_spaceport.py
View File

@@ -26,3 +26,4 @@ for cmd in cmds:
print("STDERR:", err)
client.close()

19
scripts/check_cmdline.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("cat /proc/cmdline; dmesg | grep -i 'Kernel command line'")
print("CMDLINE:")
print(stdout.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("ls -la /dev/disk/by-id/; ls -la /dev/disk/by-uuid/")
print("DISKS:")
print(stdout.read().decode('utf-8'))
client.close()

33
scripts/check_ctl03.py Normal file
View File

@@ -0,0 +1,33 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
node_ip = "140.44.4.43"
print(f"Connecting to sninfr0lnxctl03 ({node_ip})...")
client.connect(node_ip, username="root", key_filename=key_path, timeout=10)
cmds = [
"uname -r",
"ls -d /lib/modules/*",
"lsmod | grep -E 'veth|vxlan|br_netfilter'",
"modprobe veth",
"dmesg | tail -n 25"
]
for cmd in cmds:
print(f"\n>>> Running on {node_ip}: {cmd}")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8').strip()
err = stderr.read().decode('utf-8').strip()
if out:
print(out)
if err:
print("STDERR:", err)
client.close()

13
scripts/check_diff_kernel.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("cmp /boot/vmlinuz-6.12.109+rpt-rpi-2712 /boot/firmware/kernel_2712.img && echo IDENTICAL || echo DIFFERENT")
print(stdout.read().decode('utf-8'))
client.close()

13
scripts/check_dmesg_head.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("dmesg | head -n 10")
print(stdout.read().decode('utf-8'))
client.close()

13
scripts/check_dpkg_kernel.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("dpkg -l | grep -E 'linux-|raspberrypi-'")
print(stdout.read().decode('utf-8'))
client.close()

23
scripts/check_fstab_ctl03.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
cmds = [
"lsblk -f",
"cat /etc/fstab",
"mount | grep -i boot"
]
for cmd in cmds:
print(f"\n>>> Running on 140.44.4.43: {cmd}")
stdin, stdout, stderr = client.exec_command(cmd)
print(stdout.read().decode('utf-8'))
client.close()

13
scripts/check_kernel_file.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("file /boot/firmware/kernel_2712.img; ls -la /boot/vmlinuz*")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_kernel_str.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("strings /boot/firmware/kernel_2712.img | grep -E '^Linux version' | head -n 5")
print("Kernel version inside kernel_2712.img:")
print(stdout.read().decode('utf-8'))
client.close()

25
scripts/check_rpi_kernel.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
cmds = [
"cat /boot/firmware/config.txt",
"cat /boot/config.txt 2>/dev/null",
"ls -la /boot/firmware/kernel* /boot/firmware/initramfs*",
"ls -la /boot/firmware/*.img",
"dpkg -l | grep -E 'linux-image|raspberrypi-kernel'"
]
for cmd in cmds:
print(f"\n>>> Running on 140.44.4.43: {cmd}")
stdin, stdout, stderr = client.exec_command(cmd)
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_spaceport_net.py
View File

@@ -24,3 +24,4 @@ for cmd in cmds:
print(stdout.read().decode('utf-8').strip())
client.close()

15
scripts/check_veth_file.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for node, ip in [('sninfr0lnxctl01', '140.44.4.41'), ('sninfr0lnxctl03', '140.44.4.43')]:
print(f"\n=== {node} ({ip}) ===")
client.connect(ip, username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("find /lib/modules -name '*veth*'; modinfo veth 2>&1")
print(stdout.read().decode('utf-8'))
client.close()

13
scripts/list_firmware.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("ls -la /boot/firmware")
print(stdout.read().decode('utf-8'))
client.close()

63
scripts/reboot_and_verify_ctl03.py Normal file
View File

@@ -0,0 +1,63 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
target_node = "sninfr0lnxctl03"
target_ip = "140.44.4.43"
print(f"==================================================")
print(f"Rebooting {target_node} ({target_ip}) to load upgraded kernel...")
print(f"==================================================")
try:
client.connect(target_ip, username="root", key_filename=key_path, timeout=5)
print("Initiating reboot...")
client.exec_command("reboot")
client.close()
except Exception as e:
print(f"Reboot trigger exception (expected): {e}")
# Wait for node to go down
print("Waiting 15 seconds for node to shut down...")
time.sleep(15)
# Poll for node to come back up
start_time = time.time()
connected = False
while time.time() - start_time < 180:
try:
client.connect(target_ip, username="root", key_filename=key_path, timeout=3)
connected = True
print(f"\n{target_node} is back UP!")
break
except Exception:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(3)
if not connected:
print(f"\nERROR: {target_node} failed to reconnect within 180s")
sys.exit(1)
# Check kernel and modules
stdin, stdout, stderr = client.exec_command("uname -r; lsmod | grep veth || modprobe veth; lsmod | grep veth")
print("\nKernel & veth status:")
print(stdout.read().decode('utf-8'))
client.close()
# Check pods on Spaceport master
time.sleep(10)
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide")
print("\nCattle system pods on Spaceport:")
print(stdout.read().decode('utf-8'))
client.close()
print("\nSUCCESS: sninfr0lnxctl03 rebooted and verified!")

1
scripts/test_direct_41.py
View File

@@ -16,3 +16,4 @@ try:
client.close()
except Exception as e:
print("Direct connect to 140.44.4.41 error:", e)

1
scripts/test_ssh_spaceport.py
View File

@@ -16,3 +16,4 @@ print("STDOUT:", stdout.read().decode('utf-8'))
print("STDERR:", stderr.read().decode('utf-8'))
client.close()