feat: add infrastructure automation scripts and master improvement plan

scripts/inspect_ntp_status.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+Inspect the current NTP / time synchronization status across key nodes:
+- aerodrome-ctl01 (140.44.4.11)
+- nutserver (14.4.4.44)
+- gateway (140.44.4.1 / 14.4.4.1)
+- sample worker nodes
+"""
+import os
+import sys
+import socket
+import paramiko
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet")
+
+NODES = [
+    ("aerodrome-ctl01", "140.44.4.11", "root"),
+    ("nutserver",       "14.4.4.44",   "snarf"),
+    ("shipyard-swm01",  "140.44.4.71", "snarf"),
+    ("spaceport-ctl01", "140.44.4.41", "root"),
+]
+
+def check_udp_123(host):
+    # Send a simple NTP v3 client request packet (48 bytes: 0x1b followed by 47 zeros)
+    msg = b'\x1b' + 47 * b'\0'
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(2.0)
+    try:
+        sock.sendto(msg, (host, 123))
+        data, addr = sock.recvfrom(1024)
+        return True, len(data)
+    except socket.timeout:
+        return False, "timeout"
+    except Exception as e:
+        return False, str(e)
+    finally:
+        sock.close()
+
+def main():
+    print("=== Checking UDP Port 123 on Gateways ===")
+    for gw in ["140.44.4.1", "14.4.4.1", "14.10.10.1"]:
+        ok, res = check_udp_123(gw)
+        print(f"Gateway {gw}:123 -> {'RESPONSIVE (' + str(res) + ' bytes)' if ok else 'NO RESPONSE (' + str(res) + ')'}")
+
+    print("\n=== Inspecting Node OS and Time Services ===")
+    pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH)
+
+    for name, ip, user in NODES:
+        print(f"\n--- {name} ({ip}) as {user} ---")
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            client.connect(ip, username=user, pkey=pkey, timeout=10)
+            cmd = """
+            echo -n "OS: " && cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '"'
+            echo -n "Date: " && date -R
+            echo "Processes:"
+            ps aux | grep -E 'chrony|ntp|timesync' | grep -v grep || echo "  None"
+            echo "Systemd services:"
+            systemctl is-active systemd-timesyncd chrony chronyd ntp 2>&1 || true
+            """
+            stdin, stdout, stderr = client.exec_command(cmd)
+            out = stdout.read().decode('utf-8', errors='replace').strip()
+            err = stderr.read().decode('utf-8', errors='replace').strip()
+            print(out)
+            if err:
+                print("ERR:", err)
+        except Exception as e:
+            print(f"SSH Failed: {e}")
+        finally:
+            client.close()
+
+if __name__ == "__main__":
+    main()
+