91 lines
2.7 KiB
Python
91 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Monitor cold boot of sninfr2lnxwrk01 (.21) after PoE power cycle:
|
|
1. Wait for ICMP ping or TCP port 22.
|
|
2. Connect with OpenSSH via nutserver (or local paramiko).
|
|
3. Verify:
|
|
- Hostname & kernel uptime
|
|
- Filesystem mount status (rw, errors=continue)
|
|
- Internal NTP synchronization with 140.44.4.11
|
|
- k3s-agent service status
|
|
"""
|
|
import os
|
|
import sys
|
|
import time
|
|
import socket
|
|
import paramiko
|
|
|
|
sys.stdout.reconfigure(encoding="utf-8")
|
|
|
|
PRIV_KEY_PATH = os.path.expanduser("~/.ssh/id_ed25519_snarfnet")
|
|
|
|
print("=== 1. Waiting for 140.44.4.21 port 22 to open after cold boot ===")
|
|
port_open = False
|
|
for attempt in range(1, 45):
|
|
try:
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
s.settimeout(2.0)
|
|
s.connect(("140.44.4.21", 22))
|
|
s.close()
|
|
print(f"\n[PORT 22 OPEN!] Attempt {attempt}")
|
|
port_open = True
|
|
break
|
|
except Exception:
|
|
print(".", end="", flush=True)
|
|
time.sleep(2)
|
|
|
|
if not port_open:
|
|
print("\nTimeout waiting for port 22.")
|
|
sys.exit(1)
|
|
|
|
# Give Dropbear 2 seconds to initialize keys
|
|
time.sleep(2)
|
|
|
|
print("=== 2. Attempting SSH connection with id_ed25519_snarfnet via nutserver ===")
|
|
pkey = paramiko.Ed25519Key.from_private_key_file(PRIV_KEY_PATH)
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
client.connect("14.4.4.44", username="snarf", pkey=pkey, timeout=10)
|
|
|
|
ssh_cmd = """
|
|
sudo ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i /root/.ssh/id_ed25519_snarfnet root@140.44.4.21 '
|
|
echo "=== HOST & UPTIME ==="
|
|
hostname
|
|
uptime
|
|
echo "=== MOUNT STATUS ==="
|
|
findmnt -n -o TARGET,SOURCE,FSTYPE,OPTIONS /
|
|
echo "=== FILESYSTEM ERRORS BEHAVIOR ==="
|
|
tune2fs -l /dev/mmcblk0p2 | grep "Errors behavior" || true
|
|
echo "=== NTP STATUS ==="
|
|
systemctl is-active systemd-timesyncd
|
|
journalctl -u systemd-timesyncd -n 3 --no-pager | grep -iE "contacted|synchron" || true
|
|
echo "=== K3S AGENT STATUS ==="
|
|
systemctl is-active k3s-agent
|
|
'
|
|
"""
|
|
|
|
success = False
|
|
for attempt in range(1, 15):
|
|
try:
|
|
stdin, stdout, stderr = client.exec_command(ssh_cmd)
|
|
out = stdout.read().decode('utf-8', errors='replace')
|
|
err = stderr.read().decode('utf-8', errors='replace')
|
|
if "=== HOST & UPTIME ===" in out:
|
|
print(f"\n[SUCCESSFUL LOGIN on attempt {attempt}!]")
|
|
print(out)
|
|
success = True
|
|
break
|
|
else:
|
|
print(f"Attempt {attempt}: {err.strip()[:80]}")
|
|
time.sleep(3)
|
|
except Exception as e:
|
|
print(f"Attempt {attempt} exception: {e}")
|
|
time.sleep(3)
|
|
|
|
client.close()
|
|
|
|
if not success:
|
|
print("Could not complete verification via SSH.")
|
|
sys.exit(1)
|
|
|