From 14fac7151f52273e682823db03229f5c13a2746c Mon Sep 17 00:00:00 2001 From: Snarf Date: Thu, 5 Nov 2026 07:28:22 -0500 Subject: [PATCH] feat: add scripts to audit and troubleshoot spaceport kernels and cluster nodes --- scripts/audit_spaceport_kernels.py | 39 ++++++++++++++++++ scripts/check_blkid.py | 13 ++++++ scripts/check_boot_ctl03.py | 14 +++++++ scripts/check_cattle_spaceport.py | 1 + scripts/check_cmdline.py | 19 +++++++++ scripts/check_ctl03.py | 33 ++++++++++++++++ scripts/check_diff_kernel.py | 13 ++++++ scripts/check_dmesg_head.py | 13 ++++++ scripts/check_dpkg_kernel.py | 13 ++++++ scripts/check_fstab_ctl03.py | 23 +++++++++++ scripts/check_kernel_file.py | 13 ++++++ scripts/check_kernel_str.py | 14 +++++++ scripts/check_rpi_kernel.py | 25 ++++++++++++ scripts/check_spaceport_net.py | 1 + scripts/check_veth_file.py | 15 +++++++ scripts/list_firmware.py | 13 ++++++ scripts/reboot_and_verify_ctl03.py | 63 ++++++++++++++++++++++++++++++ scripts/test_direct_41.py | 1 + scripts/test_ssh_spaceport.py | 1 + 19 files changed, 327 insertions(+) create mode 100644 scripts/audit_spaceport_kernels.py create mode 100644 scripts/check_blkid.py create mode 100644 scripts/check_boot_ctl03.py create mode 100644 scripts/check_cmdline.py create mode 100644 scripts/check_ctl03.py create mode 100644 scripts/check_diff_kernel.py create mode 100644 scripts/check_dmesg_head.py create mode 100644 scripts/check_dpkg_kernel.py create mode 100644 scripts/check_fstab_ctl03.py create mode 100644 scripts/check_kernel_file.py create mode 100644 scripts/check_kernel_str.py create mode 100644 scripts/check_rpi_kernel.py create mode 100644 scripts/check_veth_file.py create mode 100644 scripts/list_firmware.py create mode 100644 scripts/reboot_and_verify_ctl03.py diff --git a/scripts/audit_spaceport_kernels.py b/scripts/audit_spaceport_kernels.py new file mode 100644 index 0000000..9bc03e8 --- /dev/null +++ b/scripts/audit_spaceport_kernels.py @@ -0,0 +1,39 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +spaceport_nodes = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44'), + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56'), +] + +print(f"{'NODE':<18} {'IP':<15} {'RUNNING KERNEL':<22} {'LATEST MODULES':<25} {'VETH LOADED':<12} {'UPTIME'}") +print("="*105) + +for name, ip in spaceport_nodes: + try: + client.connect(ip, username="root", key_filename=key_path, timeout=5) + stdin, stdout, stderr = client.exec_command("uname -r; ls -1 /lib/modules | sort -V | tail -n 1; lsmod | grep -q veth && echo YES || echo NO; uptime -p") + lines = [l.strip() for l in stdout.read().decode('utf-8').strip().split('\n') if l.strip()] + running_k = lines[0] if len(lines) > 0 else "?" + latest_mod = lines[1] if len(lines) > 1 else "?" + veth = lines[2] if len(lines) > 2 else "?" + uptime_str = lines[3] if len(lines) > 3 else "?" + print(f"{name:<18} {ip:<15} {running_k:<22} {latest_mod:<25} {veth:<12} {uptime_str}") + client.close() + except Exception as e: + print(f"{name:<18} {ip:<15} ERROR: {e}") + diff --git a/scripts/check_blkid.py b/scripts/check_blkid.py new file mode 100644 index 0000000..0611825 --- /dev/null +++ b/scripts/check_blkid.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("blkid") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/check_boot_ctl03.py b/scripts/check_boot_ctl03.py new file mode 100644 index 0000000..c5800f6 --- /dev/null +++ b/scripts/check_boot_ctl03.py @@ -0,0 +1,14 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("ls -la /boot; ls -la /boot/firmware 2>/dev/null | head -n 20") +print(stdout.read().decode('utf-8')) +client.close() + diff --git a/scripts/check_cattle_spaceport.py b/scripts/check_cattle_spaceport.py index 3b78a11..57cd66d 100644 --- a/scripts/check_cattle_spaceport.py +++ b/scripts/check_cattle_spaceport.py @@ -26,3 +26,4 @@ for cmd in cmds: print("STDERR:", err) client.close() + diff --git a/scripts/check_cmdline.py b/scripts/check_cmdline.py new file mode 100644 index 0000000..c8da842 --- /dev/null +++ b/scripts/check_cmdline.py @@ -0,0 +1,19 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("cat /proc/cmdline; dmesg | grep -i 'Kernel command line'") +print("CMDLINE:") +print(stdout.read().decode('utf-8')) + +stdin, stdout, stderr = client.exec_command("ls -la /dev/disk/by-id/; ls -la /dev/disk/by-uuid/") +print("DISKS:") +print(stdout.read().decode('utf-8')) + +client.close() diff --git a/scripts/check_ctl03.py b/scripts/check_ctl03.py new file mode 100644 index 0000000..c256513 --- /dev/null +++ b/scripts/check_ctl03.py @@ -0,0 +1,33 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +node_ip = "140.44.4.43" +print(f"Connecting to sninfr0lnxctl03 ({node_ip})...") +client.connect(node_ip, username="root", key_filename=key_path, timeout=10) + +cmds = [ + "uname -r", + "ls -d /lib/modules/*", + "lsmod | grep -E 'veth|vxlan|br_netfilter'", + "modprobe veth", + "dmesg | tail -n 25" +] + +for cmd in cmds: + print(f"\n>>> Running on {node_ip}: {cmd}") + stdin, stdout, stderr = client.exec_command(cmd) + out = stdout.read().decode('utf-8').strip() + err = stderr.read().decode('utf-8').strip() + if out: + print(out) + if err: + print("STDERR:", err) + +client.close() + diff --git a/scripts/check_diff_kernel.py b/scripts/check_diff_kernel.py new file mode 100644 index 0000000..4c9a171 --- /dev/null +++ b/scripts/check_diff_kernel.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("cmp /boot/vmlinuz-6.12.109+rpt-rpi-2712 /boot/firmware/kernel_2712.img && echo IDENTICAL || echo DIFFERENT") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/check_dmesg_head.py b/scripts/check_dmesg_head.py new file mode 100644 index 0000000..edfb4bd --- /dev/null +++ b/scripts/check_dmesg_head.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("dmesg | head -n 10") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/check_dpkg_kernel.py b/scripts/check_dpkg_kernel.py new file mode 100644 index 0000000..927c952 --- /dev/null +++ b/scripts/check_dpkg_kernel.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("dpkg -l | grep -E 'linux-|raspberrypi-'") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/check_fstab_ctl03.py b/scripts/check_fstab_ctl03.py new file mode 100644 index 0000000..02e5fed --- /dev/null +++ b/scripts/check_fstab_ctl03.py @@ -0,0 +1,23 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) + +cmds = [ + "lsblk -f", + "cat /etc/fstab", + "mount | grep -i boot" +] + +for cmd in cmds: + print(f"\n>>> Running on 140.44.4.43: {cmd}") + stdin, stdout, stderr = client.exec_command(cmd) + print(stdout.read().decode('utf-8')) + +client.close() diff --git a/scripts/check_kernel_file.py b/scripts/check_kernel_file.py new file mode 100644 index 0000000..0088066 --- /dev/null +++ b/scripts/check_kernel_file.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("file /boot/firmware/kernel_2712.img; ls -la /boot/vmlinuz*") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/check_kernel_str.py b/scripts/check_kernel_str.py new file mode 100644 index 0000000..048b070 --- /dev/null +++ b/scripts/check_kernel_str.py @@ -0,0 +1,14 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("strings /boot/firmware/kernel_2712.img | grep -E '^Linux version' | head -n 5") +print("Kernel version inside kernel_2712.img:") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/check_rpi_kernel.py b/scripts/check_rpi_kernel.py new file mode 100644 index 0000000..53ab045 --- /dev/null +++ b/scripts/check_rpi_kernel.py @@ -0,0 +1,25 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) + +cmds = [ + "cat /boot/firmware/config.txt", + "cat /boot/config.txt 2>/dev/null", + "ls -la /boot/firmware/kernel* /boot/firmware/initramfs*", + "ls -la /boot/firmware/*.img", + "dpkg -l | grep -E 'linux-image|raspberrypi-kernel'" +] + +for cmd in cmds: + print(f"\n>>> Running on 140.44.4.43: {cmd}") + stdin, stdout, stderr = client.exec_command(cmd) + print(stdout.read().decode('utf-8')) + +client.close() diff --git a/scripts/check_spaceport_net.py b/scripts/check_spaceport_net.py index edcec38..e761c93 100644 --- a/scripts/check_spaceport_net.py +++ b/scripts/check_spaceport_net.py @@ -24,3 +24,4 @@ for cmd in cmds: print(stdout.read().decode('utf-8').strip()) client.close() + diff --git a/scripts/check_veth_file.py b/scripts/check_veth_file.py new file mode 100644 index 0000000..81288a7 --- /dev/null +++ b/scripts/check_veth_file.py @@ -0,0 +1,15 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +for node, ip in [('sninfr0lnxctl01', '140.44.4.41'), ('sninfr0lnxctl03', '140.44.4.43')]: + print(f"\n=== {node} ({ip}) ===") + client.connect(ip, username="root", key_filename=key_path, timeout=10) + stdin, stdout, stderr = client.exec_command("find /lib/modules -name '*veth*'; modinfo veth 2>&1") + print(stdout.read().decode('utf-8')) + client.close() diff --git a/scripts/list_firmware.py b/scripts/list_firmware.py new file mode 100644 index 0000000..18561e2 --- /dev/null +++ b/scripts/list_firmware.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("ls -la /boot/firmware") +print(stdout.read().decode('utf-8')) +client.close() diff --git a/scripts/reboot_and_verify_ctl03.py b/scripts/reboot_and_verify_ctl03.py new file mode 100644 index 0000000..06f7101 --- /dev/null +++ b/scripts/reboot_and_verify_ctl03.py @@ -0,0 +1,63 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +target_node = "sninfr0lnxctl03" +target_ip = "140.44.4.43" + +print(f"==================================================") +print(f"Rebooting {target_node} ({target_ip}) to load upgraded kernel...") +print(f"==================================================") + +try: + client.connect(target_ip, username="root", key_filename=key_path, timeout=5) + print("Initiating reboot...") + client.exec_command("reboot") + client.close() +except Exception as e: + print(f"Reboot trigger exception (expected): {e}") + +# Wait for node to go down +print("Waiting 15 seconds for node to shut down...") +time.sleep(15) + +# Poll for node to come back up +start_time = time.time() +connected = False +while time.time() - start_time < 180: + try: + client.connect(target_ip, username="root", key_filename=key_path, timeout=3) + connected = True + print(f"\n{target_node} is back UP!") + break + except Exception: + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(3) + +if not connected: + print(f"\nERROR: {target_node} failed to reconnect within 180s") + sys.exit(1) + +# Check kernel and modules +stdin, stdout, stderr = client.exec_command("uname -r; lsmod | grep veth || modprobe veth; lsmod | grep veth") +print("\nKernel & veth status:") +print(stdout.read().decode('utf-8')) +client.close() + +# Check pods on Spaceport master +time.sleep(10) +client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10) +stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide") +print("\nCattle system pods on Spaceport:") +print(stdout.read().decode('utf-8')) +client.close() + +print("\nSUCCESS: sninfr0lnxctl03 rebooted and verified!") + diff --git a/scripts/test_direct_41.py b/scripts/test_direct_41.py index 39d19e3..960a84a 100644 --- a/scripts/test_direct_41.py +++ b/scripts/test_direct_41.py @@ -16,3 +16,4 @@ try: client.close() except Exception as e: print("Direct connect to 140.44.4.41 error:", e) + diff --git a/scripts/test_ssh_spaceport.py b/scripts/test_ssh_spaceport.py index 06809df..cd38fef 100644 --- a/scripts/test_ssh_spaceport.py +++ b/scripts/test_ssh_spaceport.py @@ -16,3 +16,4 @@ print("STDOUT:", stdout.read().decode('utf-8')) print("STDERR:", stderr.read().decode('utf-8')) client.close() +