feat: add infrastructure automation scripts and master improvement plan

scripts/wait_and_verify_21.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+Wait for sninfr2lnxwrk01 (.21) to boot, verify:
+1. Ping responsiveness
+2. Passwordless SSH access with Ed25519 key
+3. Root filesystem mount status (verify read-write)
+4. Systemd timesyncd / NTP synchronization status
+5. K3s-agent service status
+"""
+import os
+import sys
+import time
+import socket
+import paramiko
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet")
+pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH)
+
+print("=== 1. Waiting for 140.44.4.21 port 22 to become available ===")
+sock_ok = False
+for attempt in range(1, 40):
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(2.0)
+        s.connect(("140.44.4.21", 22))
+        s.close()
+        print(f"Port 22 open on attempt {attempt}!")
+        sock_ok = True
+        break
+    except Exception:
+        print(".", end="", flush=True)
+        time.sleep(3)
+
+if not sock_ok:
+    print("\nTimeout waiting for port 22.")
+    sys.exit(1)
+
+print("\n=== 2. Attempting SSH connection with id_ed25519_snarfnet ===")
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+connected = False
+for attempt in range(1, 15):
+    try:
+        ssh.connect("140.44.4.21", username="root", pkey=pkey, timeout=6, banner_timeout=15, auth_timeout=15)
+        print(f"Connected to 140.44.4.21 on attempt {attempt}!")
+        connected = True
+        break
+    except Exception as e:
+        print(f"SSH attempt {attempt} failed: {e}")
+        time.sleep(3)
+
+if not connected:
+    print("Failed to authenticate via SSH.")
+    sys.exit(1)
+
+print("\n=== 3. Checking node health on 140.44.4.21 ===")
+cmd = """
+echo "--- Hostname & Uptime ---"
+hostname; uptime
+echo "--- Root Mount Status (rw vs ro) ---"
+findmnt -n -o TARGET,SOURCE,FSTYPE,OPTIONS /
+echo "--- Timesyncd Status ---"
+systemctl is-active systemd-timesyncd 2>&1 || true
+journalctl -u systemd-timesyncd -n 3 --no-pager | grep -iE 'contacted|synchron' || true
+echo "--- K3s Agent Status ---"
+systemctl status k3s-agent --no-pager | head -n 12 || true
+"""
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode('utf-8', errors='replace'))
+ssh.close()
+