#!/usr/bin/env python3 """ Proactively harden boot & filesystem configuration across all 24 online Snarfnet RPi/Linux nodes: 1. Ensures 'fsck.repair=yes' and 'rootwait' are set in /boot/firmware/cmdline.txt (and /boot/cmdline.txt). 2. Sets ext4 root superblock error behavior to 'continue' (tune2fs -e continue). 3. Adds 'nofail,x-systemd.automount' to GlusterFS and NFS network mounts in /etc/fstab. 4. Adds 'nofail' to /boot/firmware (/boot) vfat mount in /etc/fstab so a dirty FAT bit never blocks SSH boot. 5. Reloads systemd and recovers failed network mounts on Shipyard Swarm nodes. Uses pure SSH stdin (no SFTP required) for full compatibility with DietPi Dropbear SSH. """ import base64 import sys import time import paramiko sys.stdout.reconfigure(encoding="utf-8") ONLINE_NODES = [ ("aerodrome-ctl01", "140.44.4.11", "root", "Katzir476!"), ("aerodrome-ctl02", "140.44.4.12", "root", "Katzir476!"), ("aerodrome-ctl03", "140.44.4.13", "root", "Katzir476!"), ("aerodrome-wrk01", "140.44.4.21", "root", "Katzir476!"), ("aerodrome-wrk02", "140.44.4.22", "root", "Katzir476!"), ("aerodrome-wrk03", "140.44.4.23", "root", "Katzir476!"), ("aerodrome-wrk04", "140.44.4.24", "root", "Katzir476!"), ("aerodrome-wrk05", "140.44.4.25", "root", "Katzir476!"), ("aerodrome-wrk06", "140.44.4.26", "root", "Katzir476!"), ("aerodrome-wrk07", "140.44.4.27", "root", "Katzir476!"), ("spaceport-ctl01", "140.44.4.41", "root", "dietpi"), ("spaceport-ctl02", "140.44.4.42", "root", "dietpi"), ("spaceport-ctl03", "140.44.4.43", "root", "dietpi"), ("spaceport-ctl04", "140.44.4.44", "root", "dietpi"), ("spaceport-wrk01", "140.44.4.51", "root", "dietpi"), ("spaceport-wrk02", "140.44.4.52", "root", "dietpi"), ("spaceport-wrk03", "140.44.4.53", "root", "dietpi"), ("spaceport-wrk04", "140.44.4.54", "root", "dietpi"), ("spaceport-wrk05", "140.44.4.55", "root", "Katzir476!"), ("spaceport-wrk06", "140.44.4.56", "root", "dietpi"), ("shipyard-swm01", "140.44.4.71", "snarf", "Katzir476!"), ("shipyard-swm02", "140.44.4.72", "snarf", "Katzir476!"), ("shipyard-swm03", "140.44.4.73", "snarf", "Katzir476!"), ("shipyard-swm04", "140.44.4.74", "snarf", "Katzir476!"), ("shipyard-swm05", "140.44.4.75", "snarf", "Katzir476!"), ("ai-ops01", "140.44.4.7", "snarf", "Katzir476!"), ("ai-web01", "140.44.4.8", "snarf", "Katzir476!"), ("nutserver", "14.4.4.44", "snarf", "Katzir476!"), ] HARDEN_SH = r""" set -e HOST=$(hostname) # 1. Tune root ext4 filesystem error behavior to 'continue' ROOT_DEV=$(findmnt -n -o SOURCE /) if [[ "$ROOT_DEV" == /dev/* ]]; then tune2fs -e continue "$ROOT_DEV" >/dev/null 2>&1 || true ERR_BEHAV=$(tune2fs -l "$ROOT_DEV" 2>/dev/null | grep "Errors behavior" | awk -F: '{print $2}' | xargs) echo " [EXT4] $ROOT_DEV -> Errors behavior: $ERR_BEHAV" fi # 2. Ensure fsck.repair=yes in active cmdline.txt files for cmdfile in /boot/firmware/cmdline.txt /boot/cmdline.txt; do if [ -f "$cmdfile" ] && ! grep -q "DO NOT EDIT" "$cmdfile"; then if ! grep -q "fsck.repair=yes" "$cmdfile"; then cp "$cmdfile" "${cmdfile}.bak" sed -i 's/rootwait/fsck.repair=yes rootwait/g' "$cmdfile" echo " [CMDLINE] Added fsck.repair=yes to $cmdfile" else echo " [CMDLINE] Verified fsck.repair=yes already in $cmdfile" fi fi done # 3. Harden /etc/fstab (add nofail,x-systemd.automount to glusterfs/nfs; add nofail to /boot vfat) cp /etc/fstab /etc/fstab.bak if command -v python3 >/dev/null 2>&1; then python3 -c ' with open("/etc/fstab", "r") as f: lines = f.readlines() new_lines = [] changed = False for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#"): new_lines.append(line) continue parts = stripped.split() if len(parts) >= 4: mp, fstype, opts = parts[1], parts[2], parts[3] opt_list = opts.split(",") if fstype in ("glusterfs", "nfs", "nfs4", "cifs"): for req_opt in ("_netdev", "nofail", "x-systemd.automount"): if req_opt not in opt_list: opt_list.append(req_opt) changed = True elif mp in ("/boot", "/boot/firmware") and fstype == "vfat": if "nofail" not in opt_list: opt_list.append("nofail") changed = True parts[3] = ",".join(opt_list) new_lines.append("\t".join(parts) + "\n") else: new_lines.append(line) if changed: with open("/etc/fstab", "w") as f: f.writelines(new_lines) print(" [FSTAB] Updated /etc/fstab with nofail / x-systemd.automount") else: print(" [FSTAB] Verified /etc/fstab already has nofail options") ' else if grep -E "/boot(/firmware)?[[:space:]]+vfat" /etc/fstab | grep -vq "nofail"; then sed -i -E 's|(/boot(/firmware)?[[:space:]]+vfat[[:space:]]+[^[:space:]]+)|\1,nofail|g' /etc/fstab echo " [FSTAB] Added nofail to /boot vfat in /etc/fstab (via sed)" else echo " [FSTAB] Verified /etc/fstab already has nofail on /boot" fi fi # 4. Reload systemd, reset failed units, and ensure mounts are active systemctl daemon-reload systemctl reset-failed 2>/dev/null || true if grep -q "/mnt/nfs_shares" /etc/fstab; then mount /mnt/nfs_shares 2>/dev/null || true mount /mnt/swarm_shared 2>/dev/null || true fi FAILED_CNT=$(systemctl --failed --no-legend | wc -l) echo " [SYSTEMD] Failed units count: $FAILED_CNT" """ def main(): b64_script = base64.b64encode(HARDEN_SH.encode("utf-8")).decode("ascii") for name, ip, user, pw in ONLINE_NODES: print(f"=== Hardening {name} ({ip}) ===") for attempt in range(2): try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username=user, password=pw, timeout=10, banner_timeout=15, auth_timeout=15, look_for_keys=False, allow_agent=False) if user == "root": cmd = f"echo '{b64_script}' | base64 -d | bash" else: cmd = f"echo '{pw}' | sudo -S -p '' bash -c \"echo '{b64_script}' | base64 -d | bash\"" stdin, stdout, stderr = ssh.exec_command(cmd, timeout=25) print(stdout.read().decode("utf-8", errors="ignore").strip()) err = stderr.read().decode("utf-8", errors="ignore").strip() if err: print(f" [STDERR] {err}") ssh.close() break except Exception as e: if attempt == 0: time.sleep(1.5) else: print(f" [ERROR] {e}") if __name__ == "__main__": main()