feat: add homelab topology documentation, infrastructure improvement plans, and management scripts
This commit is contained in:
172
scripts/harden_online_rpi_boot.py
Normal file
172
scripts/harden_online_rpi_boot.py
Normal file
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Proactively harden boot & filesystem configuration across all 24 online Snarfnet RPi/Linux nodes:
|
||||
1. Ensures 'fsck.repair=yes' and 'rootwait' are set in /boot/firmware/cmdline.txt (and /boot/cmdline.txt).
|
||||
2. Sets ext4 root superblock error behavior to 'continue' (tune2fs -e continue).
|
||||
3. Adds 'nofail,x-systemd.automount' to GlusterFS and NFS network mounts in /etc/fstab.
|
||||
4. Adds 'nofail' to /boot/firmware (/boot) vfat mount in /etc/fstab so a dirty FAT bit never blocks SSH boot.
|
||||
5. Reloads systemd and recovers failed network mounts on Shipyard Swarm nodes.
|
||||
Uses pure SSH stdin (no SFTP required) for full compatibility with DietPi Dropbear SSH.
|
||||
"""
|
||||
import base64
|
||||
import sys
|
||||
import time
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
ONLINE_NODES = [
|
||||
("aerodrome-ctl01", "140.44.4.11", "root", "Katzir476!"),
|
||||
("aerodrome-ctl02", "140.44.4.12", "root", "Katzir476!"),
|
||||
("aerodrome-ctl03", "140.44.4.13", "root", "Katzir476!"),
|
||||
("aerodrome-wrk01", "140.44.4.21", "root", "Katzir476!"),
|
||||
("aerodrome-wrk02", "140.44.4.22", "root", "Katzir476!"),
|
||||
("aerodrome-wrk03", "140.44.4.23", "root", "Katzir476!"),
|
||||
("aerodrome-wrk04", "140.44.4.24", "root", "Katzir476!"),
|
||||
("aerodrome-wrk05", "140.44.4.25", "root", "Katzir476!"),
|
||||
("aerodrome-wrk06", "140.44.4.26", "root", "Katzir476!"),
|
||||
("aerodrome-wrk07", "140.44.4.27", "root", "Katzir476!"),
|
||||
("spaceport-ctl01", "140.44.4.41", "root", "dietpi"),
|
||||
("spaceport-ctl02", "140.44.4.42", "root", "dietpi"),
|
||||
("spaceport-ctl03", "140.44.4.43", "root", "dietpi"),
|
||||
("spaceport-ctl04", "140.44.4.44", "root", "dietpi"),
|
||||
("spaceport-wrk01", "140.44.4.51", "root", "dietpi"),
|
||||
("spaceport-wrk02", "140.44.4.52", "root", "dietpi"),
|
||||
("spaceport-wrk03", "140.44.4.53", "root", "dietpi"),
|
||||
("spaceport-wrk04", "140.44.4.54", "root", "dietpi"),
|
||||
("spaceport-wrk05", "140.44.4.55", "root", "Katzir476!"),
|
||||
("spaceport-wrk06", "140.44.4.56", "root", "dietpi"),
|
||||
("shipyard-swm01", "140.44.4.71", "snarf", "Katzir476!"),
|
||||
("shipyard-swm02", "140.44.4.72", "snarf", "Katzir476!"),
|
||||
("shipyard-swm03", "140.44.4.73", "snarf", "Katzir476!"),
|
||||
("shipyard-swm04", "140.44.4.74", "snarf", "Katzir476!"),
|
||||
("shipyard-swm05", "140.44.4.75", "snarf", "Katzir476!"),
|
||||
("ai-ops01", "140.44.4.7", "snarf", "Katzir476!"),
|
||||
("ai-web01", "140.44.4.8", "snarf", "Katzir476!"),
|
||||
("nutserver", "14.4.4.44", "snarf", "Katzir476!"),
|
||||
]
|
||||
|
||||
HARDEN_SH = r"""
|
||||
set -e
|
||||
HOST=$(hostname)
|
||||
|
||||
# 1. Tune root ext4 filesystem error behavior to 'continue'
|
||||
ROOT_DEV=$(findmnt -n -o SOURCE /)
|
||||
if [[ "$ROOT_DEV" == /dev/* ]]; then
|
||||
tune2fs -e continue "$ROOT_DEV" >/dev/null 2>&1 || true
|
||||
ERR_BEHAV=$(tune2fs -l "$ROOT_DEV" 2>/dev/null | grep "Errors behavior" | awk -F: '{print $2}' | xargs)
|
||||
echo " [EXT4] $ROOT_DEV -> Errors behavior: $ERR_BEHAV"
|
||||
fi
|
||||
|
||||
# 2. Ensure fsck.repair=yes (and usb-storage.quirks=152d:0583:u if JMicron bridge present) in active cmdline.txt files
|
||||
HAS_JMICRON=0
|
||||
if lsusb 2>/dev/null | grep -qi "152d:0583"; then
|
||||
HAS_JMICRON=1
|
||||
fi
|
||||
for cmdfile in /boot/firmware/cmdline.txt /boot/cmdline.txt; do
|
||||
if [ -f "$cmdfile" ] && ! grep -q "DO NOT EDIT" "$cmdfile"; then
|
||||
if ! grep -q "fsck.repair=yes" "$cmdfile"; then
|
||||
cp "$cmdfile" "${cmdfile}.bak"
|
||||
sed -i 's/rootwait/fsck.repair=yes rootwait/g' "$cmdfile"
|
||||
echo " [CMDLINE] Added fsck.repair=yes to $cmdfile"
|
||||
else
|
||||
echo " [CMDLINE] Verified fsck.repair=yes already in $cmdfile"
|
||||
fi
|
||||
if [ "$HAS_JMICRON" -eq 1 ] && ! grep -q "usb-storage.quirks=152d:0583:u" "$cmdfile"; then
|
||||
cp "$cmdfile" "${cmdfile}.bak"
|
||||
sed -i 's/^/usb-storage.quirks=152d:0583:u /' "$cmdfile"
|
||||
echo " [CMDLINE] Added usb-storage.quirks=152d:0583:u to $cmdfile"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# 3. Harden /etc/fstab (add nofail,x-systemd.automount to glusterfs/nfs; add nofail to /boot vfat)
|
||||
cp /etc/fstab /etc/fstab.bak
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
python3 -c '
|
||||
with open("/etc/fstab", "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
new_lines = []
|
||||
changed = False
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
new_lines.append(line)
|
||||
continue
|
||||
parts = stripped.split()
|
||||
if len(parts) >= 4:
|
||||
mp, fstype, opts = parts[1], parts[2], parts[3]
|
||||
opt_list = opts.split(",")
|
||||
if fstype in ("glusterfs", "nfs", "nfs4", "cifs"):
|
||||
for req_opt in ("_netdev", "nofail", "x-systemd.automount"):
|
||||
if req_opt not in opt_list:
|
||||
opt_list.append(req_opt)
|
||||
changed = True
|
||||
elif mp in ("/boot", "/boot/firmware") and fstype == "vfat":
|
||||
if "nofail" not in opt_list:
|
||||
opt_list.append("nofail")
|
||||
changed = True
|
||||
parts[3] = ",".join(opt_list)
|
||||
new_lines.append("\t".join(parts) + "\n")
|
||||
else:
|
||||
new_lines.append(line)
|
||||
|
||||
if changed:
|
||||
with open("/etc/fstab", "w") as f:
|
||||
f.writelines(new_lines)
|
||||
print(" [FSTAB] Updated /etc/fstab with nofail / x-systemd.automount")
|
||||
else:
|
||||
print(" [FSTAB] Verified /etc/fstab already has nofail options")
|
||||
'
|
||||
else
|
||||
if grep -E "/boot(/firmware)?[[:space:]]+vfat" /etc/fstab | grep -vq "nofail"; then
|
||||
sed -i -E 's|(/boot(/firmware)?[[:space:]]+vfat[[:space:]]+[^[:space:]]+)|\1,nofail|g' /etc/fstab
|
||||
echo " [FSTAB] Added nofail to /boot vfat in /etc/fstab (via sed)"
|
||||
else
|
||||
echo " [FSTAB] Verified /etc/fstab already has nofail on /boot"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Reload systemd, reset failed units, and ensure mounts are active
|
||||
systemctl daemon-reload
|
||||
systemctl reset-failed 2>/dev/null || true
|
||||
if grep -q "/mnt/nfs_shares" /etc/fstab; then
|
||||
mount /mnt/nfs_shares 2>/dev/null || true
|
||||
mount /mnt/swarm_shared 2>/dev/null || true
|
||||
fi
|
||||
FAILED_CNT=$(systemctl --failed --no-legend | wc -l)
|
||||
echo " [SYSTEMD] Failed units count: $FAILED_CNT"
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
b64_script = base64.b64encode(HARDEN_SH.encode("utf-8")).decode("ascii")
|
||||
for name, ip, user, pw in ONLINE_NODES:
|
||||
print(f"=== Hardening {name} ({ip}) ===")
|
||||
for attempt in range(2):
|
||||
try:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username=user, password=pw, timeout=10, banner_timeout=15, auth_timeout=15, look_for_keys=False, allow_agent=False)
|
||||
if user == "root":
|
||||
cmd = f"echo '{b64_script}' | base64 -d | bash"
|
||||
else:
|
||||
cmd = f"echo '{pw}' | sudo -S -p '' bash -c \"echo '{b64_script}' | base64 -d | bash\""
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=25)
|
||||
print(stdout.read().decode("utf-8", errors="ignore").strip())
|
||||
err = stderr.read().decode("utf-8", errors="ignore").strip()
|
||||
if err:
|
||||
print(f" [STDERR] {err}")
|
||||
ssh.close()
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 0:
|
||||
time.sleep(1.5)
|
||||
else:
|
||||
print(f" [ERROR] {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user