164 lines
6.7 KiB
Python
164 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Proactively harden boot & filesystem configuration across all 24 online Snarfnet RPi/Linux nodes:
|
|
1. Ensures 'fsck.repair=yes' and 'rootwait' are set in /boot/firmware/cmdline.txt (and /boot/cmdline.txt).
|
|
2. Sets ext4 root superblock error behavior to 'continue' (tune2fs -e continue).
|
|
3. Adds 'nofail,x-systemd.automount' to GlusterFS and NFS network mounts in /etc/fstab.
|
|
4. Adds 'nofail' to /boot/firmware (/boot) vfat mount in /etc/fstab so a dirty FAT bit never blocks SSH boot.
|
|
5. Reloads systemd and recovers failed network mounts on Shipyard Swarm nodes.
|
|
Uses pure SSH stdin (no SFTP required) for full compatibility with DietPi Dropbear SSH.
|
|
"""
|
|
import base64
|
|
import sys
|
|
import time
|
|
import paramiko
|
|
|
|
sys.stdout.reconfigure(encoding="utf-8")
|
|
|
|
ONLINE_NODES = [
|
|
("aerodrome-ctl01", "140.44.4.11", "root", "Katzir476!"),
|
|
("aerodrome-ctl02", "140.44.4.12", "root", "Katzir476!"),
|
|
("aerodrome-ctl03", "140.44.4.13", "root", "Katzir476!"),
|
|
("aerodrome-wrk01", "140.44.4.21", "root", "Katzir476!"),
|
|
("aerodrome-wrk02", "140.44.4.22", "root", "Katzir476!"),
|
|
("aerodrome-wrk03", "140.44.4.23", "root", "Katzir476!"),
|
|
("aerodrome-wrk04", "140.44.4.24", "root", "Katzir476!"),
|
|
("aerodrome-wrk05", "140.44.4.25", "root", "Katzir476!"),
|
|
("aerodrome-wrk06", "140.44.4.26", "root", "Katzir476!"),
|
|
("aerodrome-wrk07", "140.44.4.27", "root", "Katzir476!"),
|
|
("spaceport-ctl01", "140.44.4.41", "root", "dietpi"),
|
|
("spaceport-ctl02", "140.44.4.42", "root", "dietpi"),
|
|
("spaceport-ctl03", "140.44.4.43", "root", "dietpi"),
|
|
("spaceport-ctl04", "140.44.4.44", "root", "dietpi"),
|
|
("spaceport-wrk01", "140.44.4.51", "root", "dietpi"),
|
|
("spaceport-wrk02", "140.44.4.52", "root", "dietpi"),
|
|
("spaceport-wrk03", "140.44.4.53", "root", "dietpi"),
|
|
("spaceport-wrk04", "140.44.4.54", "root", "dietpi"),
|
|
("spaceport-wrk05", "140.44.4.55", "root", "Katzir476!"),
|
|
("spaceport-wrk06", "140.44.4.56", "root", "dietpi"),
|
|
("shipyard-swm01", "140.44.4.71", "snarf", "Katzir476!"),
|
|
("shipyard-swm02", "140.44.4.72", "snarf", "Katzir476!"),
|
|
("shipyard-swm03", "140.44.4.73", "snarf", "Katzir476!"),
|
|
("shipyard-swm04", "140.44.4.74", "snarf", "Katzir476!"),
|
|
("shipyard-swm05", "140.44.4.75", "snarf", "Katzir476!"),
|
|
("ai-ops01", "140.44.4.7", "snarf", "Katzir476!"),
|
|
("ai-web01", "140.44.4.8", "snarf", "Katzir476!"),
|
|
("nutserver", "14.4.4.44", "snarf", "Katzir476!"),
|
|
]
|
|
|
|
HARDEN_SH = r"""
|
|
set -e
|
|
HOST=$(hostname)
|
|
|
|
# 1. Tune root ext4 filesystem error behavior to 'continue'
|
|
ROOT_DEV=$(findmnt -n -o SOURCE /)
|
|
if [[ "$ROOT_DEV" == /dev/* ]]; then
|
|
tune2fs -e continue "$ROOT_DEV" >/dev/null 2>&1 || true
|
|
ERR_BEHAV=$(tune2fs -l "$ROOT_DEV" 2>/dev/null | grep "Errors behavior" | awk -F: '{print $2}' | xargs)
|
|
echo " [EXT4] $ROOT_DEV -> Errors behavior: $ERR_BEHAV"
|
|
fi
|
|
|
|
# 2. Ensure fsck.repair=yes in active cmdline.txt files
|
|
for cmdfile in /boot/firmware/cmdline.txt /boot/cmdline.txt; do
|
|
if [ -f "$cmdfile" ] && ! grep -q "DO NOT EDIT" "$cmdfile"; then
|
|
if ! grep -q "fsck.repair=yes" "$cmdfile"; then
|
|
cp "$cmdfile" "${cmdfile}.bak"
|
|
sed -i 's/rootwait/fsck.repair=yes rootwait/g' "$cmdfile"
|
|
echo " [CMDLINE] Added fsck.repair=yes to $cmdfile"
|
|
else
|
|
echo " [CMDLINE] Verified fsck.repair=yes already in $cmdfile"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# 3. Harden /etc/fstab (add nofail,x-systemd.automount to glusterfs/nfs; add nofail to /boot vfat)
|
|
cp /etc/fstab /etc/fstab.bak
|
|
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 -c '
|
|
with open("/etc/fstab", "r") as f:
|
|
lines = f.readlines()
|
|
|
|
new_lines = []
|
|
changed = False
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
if not stripped or stripped.startswith("#"):
|
|
new_lines.append(line)
|
|
continue
|
|
parts = stripped.split()
|
|
if len(parts) >= 4:
|
|
mp, fstype, opts = parts[1], parts[2], parts[3]
|
|
opt_list = opts.split(",")
|
|
if fstype in ("glusterfs", "nfs", "nfs4", "cifs"):
|
|
for req_opt in ("_netdev", "nofail", "x-systemd.automount"):
|
|
if req_opt not in opt_list:
|
|
opt_list.append(req_opt)
|
|
changed = True
|
|
elif mp in ("/boot", "/boot/firmware") and fstype == "vfat":
|
|
if "nofail" not in opt_list:
|
|
opt_list.append("nofail")
|
|
changed = True
|
|
parts[3] = ",".join(opt_list)
|
|
new_lines.append("\t".join(parts) + "\n")
|
|
else:
|
|
new_lines.append(line)
|
|
|
|
if changed:
|
|
with open("/etc/fstab", "w") as f:
|
|
f.writelines(new_lines)
|
|
print(" [FSTAB] Updated /etc/fstab with nofail / x-systemd.automount")
|
|
else:
|
|
print(" [FSTAB] Verified /etc/fstab already has nofail options")
|
|
'
|
|
else
|
|
if grep -E "/boot(/firmware)?[[:space:]]+vfat" /etc/fstab | grep -vq "nofail"; then
|
|
sed -i -E 's|(/boot(/firmware)?[[:space:]]+vfat[[:space:]]+[^[:space:]]+)|\1,nofail|g' /etc/fstab
|
|
echo " [FSTAB] Added nofail to /boot vfat in /etc/fstab (via sed)"
|
|
else
|
|
echo " [FSTAB] Verified /etc/fstab already has nofail on /boot"
|
|
fi
|
|
fi
|
|
|
|
# 4. Reload systemd, reset failed units, and ensure mounts are active
|
|
systemctl daemon-reload
|
|
systemctl reset-failed 2>/dev/null || true
|
|
if grep -q "/mnt/nfs_shares" /etc/fstab; then
|
|
mount /mnt/nfs_shares 2>/dev/null || true
|
|
mount /mnt/swarm_shared 2>/dev/null || true
|
|
fi
|
|
FAILED_CNT=$(systemctl --failed --no-legend | wc -l)
|
|
echo " [SYSTEMD] Failed units count: $FAILED_CNT"
|
|
"""
|
|
|
|
|
|
def main():
|
|
b64_script = base64.b64encode(HARDEN_SH.encode("utf-8")).decode("ascii")
|
|
for name, ip, user, pw in ONLINE_NODES:
|
|
print(f"=== Hardening {name} ({ip}) ===")
|
|
for attempt in range(2):
|
|
try:
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
ssh.connect(ip, username=user, password=pw, timeout=10, banner_timeout=15, auth_timeout=15, look_for_keys=False, allow_agent=False)
|
|
if user == "root":
|
|
cmd = f"echo '{b64_script}' | base64 -d | bash"
|
|
else:
|
|
cmd = f"echo '{pw}' | sudo -S -p '' bash -c \"echo '{b64_script}' | base64 -d | bash\""
|
|
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=25)
|
|
print(stdout.read().decode("utf-8", errors="ignore").strip())
|
|
err = stderr.read().decode("utf-8", errors="ignore").strip()
|
|
if err:
|
|
print(f" [STDERR] {err}")
|
|
ssh.close()
|
|
break
|
|
except Exception as e:
|
|
if attempt == 0:
|
|
time.sleep(1.5)
|
|
else:
|
|
print(f" [ERROR] {e}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|