feat: add infrastructure automation scripts and master improvement plan

scripts/monitor_21_cold_boot.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Monitor cold boot of sninfr2lnxwrk01 (.21) after PoE power cycle:
+1. Wait for ICMP ping or TCP port 22.
+2. Connect with OpenSSH via nutserver (or local paramiko).
+3. Verify:
+   - Hostname & kernel uptime
+   - Filesystem mount status (rw, errors=continue)
+   - Internal NTP synchronization with 140.44.4.11
+   - k3s-agent service status
+"""
+import os
+import sys
+import time
+import socket
+import paramiko
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet")
+
+print("=== 1. Waiting for 140.44.4.21 port 22 to open after cold boot ===")
+port_open = False
+for attempt in range(1, 45):
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(2.0)
+        s.connect(("140.44.4.21", 22))
+        s.close()
+        print(f"\n[PORT 22 OPEN!] Attempt {attempt}")
+        port_open = True
+        break
+    except Exception:
+        print(".", end="", flush=True)
+        time.sleep(2)
+
+if not port_open:
+    print("\nTimeout waiting for port 22.")
+    sys.exit(1)
+
+# Give Dropbear 2 seconds to initialize keys
+time.sleep(2)
+
+print("=== 2. Attempting SSH connection with id_ed25519_snarfnet via nutserver ===")
+pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH)
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("14.4.4.44", username="snarf", pkey=pkey, timeout=10)
+
+ssh_cmd = """
+sudo ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i /root/.ssh/id_ed25519_snarfnet root@140.44.4.21 '
+    echo "=== HOST & UPTIME ==="
+    hostname
+    uptime
+    echo "=== MOUNT STATUS ==="
+    findmnt -n -o TARGET,SOURCE,FSTYPE,OPTIONS /
+    echo "=== FILESYSTEM ERRORS BEHAVIOR ==="
+    tune2fs -l /dev/mmcblk0p2 | grep "Errors behavior" || true
+    echo "=== NTP STATUS ==="
+    systemctl is-active systemd-timesyncd
+    journalctl -u systemd-timesyncd -n 3 --no-pager | grep -iE "contacted|synchron" || true
+    echo "=== K3S AGENT STATUS ==="
+    systemctl is-active k3s-agent
+'
+"""
+
+success = False
+for attempt in range(1, 15):
+    try:
+        stdin, stdout, stderr = client.exec_command(ssh_cmd)
+        out = stdout.read().decode('utf-8', errors='replace')
+        err = stderr.read().decode('utf-8', errors='replace')
+        if "=== HOST & UPTIME ===" in out:
+            print(f"\n[SUCCESSFUL LOGIN on attempt {attempt}!]")
+            print(out)
+            success = True
+            break
+        else:
+            print(f"Attempt {attempt}: {err.strip()[:80]}")
+            time.sleep(3)
+    except Exception as e:
+        print(f"Attempt {attempt} exception: {e}")
+        time.sleep(3)
+
+client.close()
+
+if not success:
+    print("Could not complete verification via SSH.")
+    sys.exit(1)
+