feat: add boot staging configuration and automation utility scripts

boot_staging/Automation_Custom_Script.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+echo "[FIRST-BOOT] Starting automated provisioning for sninfr2lnxwrk01..."
+
+# 1. SSH Key Enrollment
+PUB_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOsRiwuL7nfm1Ruew+Xk+3tbzD9ONyKAO0q2QwkppEPZ snarfnet-automation-master@snarfnet.net"
+mkdir -p /root/.ssh /etc/dropbear
+chmod 700 /root/.ssh
+echo "$PUB_KEY" >> /root/.ssh/authorized_keys
+echo "$PUB_KEY" >> /etc/dropbear/authorized_keys
+chmod 600 /root/.ssh/authorized_keys /etc/dropbear/authorized_keys
+
+# 2. Filesystem Hardening
+ROOT_DEV=$(findmnt -n -o SOURCE /)
+if [[ "$ROOT_DEV" == /dev/* ]]; then
+    tune2fs -e continue "$ROOT_DEV" || true
+fi
+
+# 3. Time Synchronization
+cat << 'EOF' > /etc/systemd/timesyncd.conf
+[Time]
+NTP=140.44.4.11 14.4.4.44
+FallbackNTP=time.cloudflare.com time.google.com 0.us.pool.ntp.org 1.us.pool.ntp.org
+PollIntervalMinSec=32
+PollIntervalMaxSec=512
+EOF
+systemctl restart systemd-timesyncd.service || true
+
+# 4. Install & Configure Zabbix Agent 2
+apt-get update -y
+apt-get install -y zabbix-agent2
+mkdir -p /etc/zabbix
+echo "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0" > /etc/zabbix/zabbix_agent2.psk
+chmod 600 /etc/zabbix/zabbix_agent2.psk
+chown zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true
+
+cat << 'EOF' > /etc/zabbix/zabbix_agent2.conf
+Server=zabbix.snarfnet.net
+ServerActive=zabbix.snarfnet.net
+HostnameItem=system.hostname
+HostMetadata=Linux
+TLSConnect=psk
+TLSAccept=psk
+TLSPSKIdentity=PSK_autoregister
+TLSPSKFile=/etc/zabbix/zabbix_agent2.psk
+LogFile=/var/log/zabbix/zabbix_agent2.log
+LogFileSize=10
+BufferSend=5
+BufferSize=100
+EOF
+systemctl enable --now zabbix-agent2.service
+
+# 5. Install K3s Agent
+curl -sfL https://get.k3s.io | K3S_URL=https://140.44.4.11:6443 K3S_TOKEN=myworstkeptsecret44 sh -
+
+echo "[FIRST-BOOT] Provisioning completed successfully!"
+

boot_staging/dietpi.txt

@@ -0,0 +1,44 @@
+# Standard DietPi Automated Headless Config for sninfr2lnxwrk01 (140.44.4.21)
+AUTO_SETUP_LOCALE=C.UTF-8
+AUTO_SETUP_KEYBOARD_LAYOUT=us
+AUTO_SETUP_TIMEZONE=America/New_York
+AUTO_SETUP_NET_ETHERNET_ENABLED=1
+AUTO_SETUP_NET_WIFI_ENABLED=0
+AUTO_SETUP_NET_USESTATIC=1
+AUTO_SETUP_NET_STATIC_IP=140.44.4.21
+AUTO_SETUP_NET_STATIC_MASK=255.255.255.0
+AUTO_SETUP_NET_STATIC_GATEWAY=140.44.4.1
+AUTO_SETUP_NET_STATIC_DNS=140.44.4.11 14.4.4.44
+AUTO_SETUP_DHCP_TO_STATIC=0
+AUTO_SETUP_NET_HOSTNAME=sninfr2lnxwrk01
+AUTO_SETUP_NET_ETH_FORCE_SPEED=0
+AUTO_SETUP_BOOT_WAIT_FOR_NETWORK=1
+AUTO_SETUP_SWAPFILE_SIZE=0
+AUTO_SETUP_HEADLESS=1
+AUTO_UNMASK_LOGIND=0
+AUTO_SETUP_CUSTOM_SCRIPT_EXEC=1
+AUTO_SETUP_BACKUP_RESTORE=0
+AUTO_SETUP_SSH_SERVER_INDEX=-1
+AUTO_SETUP_LOGGING_INDEX=-1
+AUTO_SETUP_RAMLOG_MAXSIZE=50
+AUTO_SETUP_WEB_SERVER_INDEX=0
+AUTO_SETUP_DESKTOP_INDEX=0
+AUTO_SETUP_BROWSER_INDEX=-1
+AUTO_SETUP_AUTOSTART_TARGET_INDEX=0
+AUTO_SETUP_AUTOSTART_LOGIN_USER=root
+AUTO_SETUP_AUTOMATED=1
+AUTO_SETUP_GLOBAL_PASSWORD=dietpi
+SURVEY_OPTED_IN=0
+CONFIG_CPU_GOVERNOR=performance
+CONFIG_CHECK_DIETPI_UPDATES=1
+CONFIG_CHECK_APT_UPDATES=1
+CONFIG_NTP_MODE=4
+CONFIG_SERIAL_CONSOLE_ENABLE=1
+CONFIG_SOUNDCARD=none
+CONFIG_LCDPANEL=none
+CONFIG_ENABLE_IPV6=0
+CONFIG_NTP_MIRROR=default
+SOFTWARE_DISABLE_SSH_PASSWORD_LOGINS=0
+DEV_GITBRANCH=master
+DEV_GITOWNER=MichaIng
+

scripts/assign_letter.txt

@@ -0,0 +1,4 @@
+select disk 4
+select partition 1
+assign letter=H
+

scripts/check_22_k3s_env.py

@@ -0,0 +1,17 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.22', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command('cat /etc/systemd/system/k3s-agent.service.env')
+print('K3S ENV:\n', stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command('cat /boot/cmdline.txt /boot/firmware/cmdline.txt 2>/dev/null')
+print('CMDLINE:\n', stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command('cat /boot/config.txt /boot/firmware/config.txt 2>/dev/null')
+print('CONFIG.TXT (TAIL 20):\n', stdout.read().decode().splitlines()[-20:])
+
+ssh.close()
+

scripts/check_22_os.py

@@ -0,0 +1,21 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.22', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmds = [
+    "cat /etc/os-release",
+    "cat /boot/dietpi/.version 2>/dev/null || cat /boot/firmware/dietpi/.version 2>/dev/null || echo 'Not DietPi'",
+    "which dropbear || which sshd",
+    "ip route show",
+    "cat /etc/network/interfaces /etc/dhcpcd.conf /etc/netplan/* /etc/systemd/network/* 2>/dev/null",
+    "systemctl cat k3s-agent.service 2>/dev/null || cat /etc/systemd/system/k3s-agent.service"
+]
+
+for cmd in cmds:
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+
+ssh.close()

scripts/check_dietpi_progress.py

@@ -0,0 +1,22 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.21', username='root', password='dietpi', timeout=10)
+
+cmds = [
+    "uptime",
+    "cat /boot/dietpi/.version 2>/dev/null || cat /boot/dietpi.txt | grep -E 'AUTO_|CONFIG_'",
+    "ps aux",
+    "ls -la /boot/Automation_Custom_Script.sh /boot/Automation_Custom_PreScript.sh 2>/dev/null",
+    "ls -la /var/tmp/dietpi/logs/",
+    "cat /var/tmp/dietpi/logs/dietpi-firstrun-setup.log 2>/dev/null | tail -n 30"
+]
+
+for cmd in cmds:
+    print(f"\n=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+
+ssh.close()
+

scripts/check_images_21.py

@@ -0,0 +1,13 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("kubectl get pods -A --field-selector spec.nodeName=sninfr2lnxwrk01 -o jsonpath='{range .items[*]}{.spec.containers[*].image}{\"\\n\"}{end}'")
+images = set(stdout.read().decode().split())
+for img in sorted(images):
+    print("CACHED:", img)
+
+ssh.close()
+

scripts/check_job_100.py

@@ -0,0 +1,20 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Job
+job = Job.objects.get(id=100)
+print(f"Job [100]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+events = job.job_events.order_by('id')
+for e in events:
+    if e.stdout:
+        print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()

scripts/debug_worker_21.py

@@ -0,0 +1,18 @@
+import paramiko, time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = 'kubectl run debug-21 --image=busybox --restart=Never --overrides=\'{"spec":{"nodeName":"sninfr2lnxwrk01","hostPID":true,"hostIPC":true,"hostNetwork":true,"containers":[{"name":"debug","image":"busybox","command":["nsenter","-t","1","-m","-u","-i","-n","-p","sh","-c","ls -la /root/.ssh; cat /root/.ssh/authorized_keys"],"securityContext":{"privileged":true}}]}}\''
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print('RUN:', stdout.read().decode())
+print('RUN ERR:', stderr.read().decode())
+
+time.sleep(6)
+stdin, stdout, stderr = ssh.exec_command('kubectl logs debug-21')
+print('LOGS:\n', stdout.read().decode())
+
+ssh.exec_command('kubectl delete pod debug-21 --now')
+ssh.close()
+

scripts/deploy_sd_config.py

@@ -0,0 +1,60 @@
+import os
+import sys
+import shutil
+
+BOOT_DRIVE = "H:\\"
+STAGING_DIR = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\boot_staging"
+
+print(f"Checking for DietPi boot filesystem on {BOOT_DRIVE}...")
+
+if not os.path.exists(BOOT_DRIVE):
+    print(f"Error: {BOOT_DRIVE} is not mounted.")
+    sys.exit(1)
+
+# Check if config.txt or dietpi.txt exists on H:\
+boot_indicators = ["config.txt", "dietpi.txt", "cmdline.txt"]
+found = [f for f in boot_indicators if os.path.exists(os.path.join(BOOT_DRIVE, f))]
+
+if not found:
+    print(f"Warning: {BOOT_DRIVE} does not look like a freshly flashed DietPi boot partition yet (indicators not found).")
+    print(f"Current contents of {BOOT_DRIVE}: {os.listdir(BOOT_DRIVE)}")
+    sys.exit(2)
+
+print(f"Found boot filesystem! Detected: {found}")
+
+# 1. Copy dietpi.txt
+src_dietpi = os.path.join(STAGING_DIR, "dietpi.txt")
+dst_dietpi = os.path.join(BOOT_DRIVE, "dietpi.txt")
+shutil.copy2(src_dietpi, dst_dietpi)
+print(f"  [OK] Injected pre-configured {dst_dietpi}")
+
+# 2. Copy Automation_Custom_Script.sh
+src_script = os.path.join(STAGING_DIR, "Automation_Custom_Script.sh")
+dst_script = os.path.join(BOOT_DRIVE, "Automation_Custom_Script.sh")
+shutil.copy2(src_script, dst_script)
+print(f"  [OK] Injected first-boot {dst_script}")
+
+# 3. Patch cmdline.txt with cgroups
+cmdline_path = os.path.join(BOOT_DRIVE, "cmdline.txt")
+if os.path.exists(cmdline_path):
+    with open(cmdline_path, "r", encoding="utf-8") as f:
+        cmdline = f.read().strip()
+    
+    cgroups = "cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory net.ifnames=0"
+    if "cgroup_memory=1" not in cmdline:
+        cmdline = f"{cmdline} {cgroups}\n"
+        with open(cmdline_path, "w", encoding="utf-8", newline="\n") as f:
+            f.write(cmdline)
+        print("  [OK] Patched cmdline.txt with cgroup parameters")
+    else:
+        print("  [OK] cmdline.txt already contains cgroup parameters")
+
+# 4. Create empty 'ssh' file if needed
+ssh_flag = os.path.join(BOOT_DRIVE, "ssh")
+with open(ssh_flag, "w") as f:
+    pass
+print("  [OK] Created empty ssh flag file")
+
+print("\n=== SD Card Configuration Complete! ===")
+print("The card is fully prepared to boot unattended into sninfr2lnxwrk01, join Aerodrome K3s, sync time, and register with Zabbix!")
+

scripts/deploy_zabbix_21.py

@@ -0,0 +1,46 @@
+import sys, paramiko, time
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+script = """#!/bin/bash
+set -e
+
+mkdir -p /etc/zabbix
+echo "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0" > /etc/zabbix/zabbix_agent2.psk
+chmod 600 /etc/zabbix/zabbix_agent2.psk
+chown zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true
+
+cat << 'EOF' > /etc/zabbix/zabbix_agent2.conf
+Server=zabbix.snarfnet.net
+ServerActive=zabbix.snarfnet.net
+HostnameItem=system.hostname
+HostMetadata=Linux
+TLSConnect=psk
+TLSAccept=psk
+TLSPSKIdentity=PSK_autoregister
+TLSPSKFile=/etc/zabbix/zabbix_agent2.psk
+LogFile=/var/log/zabbix/zabbix_agent2.log
+LogFileSize=10
+BufferSend=5
+BufferSize=100
+EOF
+
+systemctl enable --now zabbix-agent2.service
+systemctl restart zabbix-agent2.service
+sleep 2
+systemctl is-active zabbix-agent2.service
+"""
+
+stdin, stdout, stderr = ssh.exec_command(script)
+print("OUT:", stdout.read().decode('utf-8', errors='replace').strip())
+print("ERR:", stderr.read().decode('utf-8', errors='replace').strip())
+
+time.sleep(3)
+stdin, stdout, stderr = ssh.exec_command("tail -n 15 /var/log/zabbix/zabbix_agent2.log")
+print("\nLOG:\n", stdout.read().decode('utf-8', errors='replace'))
+
+ssh.close()
+

scripts/detect_sd_drive.py

@@ -0,0 +1,6 @@
+import subprocess, json
+
+cmd = ["powershell", "-NoProfile", "-Command", "Get-Partition | Select-Object DiskNumber, PartitionNumber, DriveLetter, Size, Type | ConvertTo-Json"]
+res = subprocess.run(cmd, capture_output=True, text=True)
+print(res.stdout)
+

scripts/find_gitea_repo.py

@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("kubectl exec -n infrastructure deployment/gitea -c gitea -- find / -name '*snarfcode.git*' 2>/dev/null")
+print("FOUND:", stdout.read().decode())
+
+ssh.close()
+

scripts/fix_and_finish_21.py

@@ -0,0 +1,47 @@
+import paramiko, sys
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.21', username='root', password='dietpi', timeout=10)
+print("[OK] Connected to 140.44.4.21 via SSH")
+
+# 1. Fix DNS in /etc/resolv.conf and /boot/dietpi.txt
+fix_dns_cmd = """
+echo "nameserver 140.44.4.1" > /etc/resolv.conf
+echo "nameserver 1.1.1.1" >> /etc/resolv.conf
+sed -i 's/^AUTO_SETUP_NET_STATIC_DNS=.*/AUTO_SETUP_NET_STATIC_DNS=140.44.4.1 1.1.1.1/' /boot/dietpi.txt
+getent hosts get.k3s.io
+"""
+stdin, stdout, stderr = ssh.exec_command(fix_dns_cmd)
+print("DNS resolution test:", stdout.read().decode().strip())
+
+# 2. Kill stuck whiptail / login process
+kill_cmd = "pkill -9 whiptail || true; pkill -9 dietpi-login || true"
+stdin, stdout, stderr = ssh.exec_command(kill_cmd)
+print("Killed stuck first-run whiptail UI")
+
+# 3. Inject SSH Key immediately
+pub_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOsRiwuL7nfm1Ruew+Xk+3tbzD9ONyKAO0q2QwkppEPZ snarfnet-automation-master@snarfnet.net"
+ssh_inject = f"""
+mkdir -p /root/.ssh /etc/dropbear
+echo "{pub_key}" >> /root/.ssh/authorized_keys
+echo "{pub_key}" >> /etc/dropbear/authorized_keys
+chmod 700 /root/.ssh
+chmod 600 /root/.ssh/authorized_keys /etc/dropbear/authorized_keys
+"""
+stdin, stdout, stderr = ssh.exec_command(ssh_inject)
+print("[OK] Injected SSH automation master key into authorized_keys")
+
+# 4. Run /boot/Automation_Custom_Script.sh
+print("Executing /boot/Automation_Custom_Script.sh...")
+stdin, stdout, stderr = ssh.exec_command("bash /boot/Automation_Custom_Script.sh")
+
+# Stream output in real time
+for line in iter(stdout.readline, ""):
+    print(line, end="")
+
+print("\nSTDERR:")
+print(stderr.read().decode())
+
+ssh.close()
+

scripts/fix_ssh_on_21.py

@@ -0,0 +1,57 @@
+import paramiko, time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+pub_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOsRiwuL7nfm1Ruew+Xk+3tbzD9ONyKAO0q2QwkppEPZ snarfnet-automation-master@snarfnet.net"
+
+pod_manifest = f"""
+apiVersion: v1
+kind: Pod
+metadata:
+  name: fix-ssh-21
+  namespace: kube-system
+spec:
+  nodeName: sninfr2lnxwrk01
+  restartPolicy: Never
+  containers:
+  - name: fixer
+    image: busybox
+    imagePullPolicy: IfNotPresent
+    command: ["sh", "-c"]
+    args:
+    - |
+      mkdir -p /host/root/.ssh
+      chmod 700 /host/root/.ssh
+      echo "{pub_key}" > /host/root/.ssh/authorized_keys
+      chmod 600 /host/root/.ssh/authorized_keys
+      mkdir -p /host/etc/dropbear
+      echo "{pub_key}" > /host/etc/dropbear/authorized_keys
+      chmod 600 /host/etc/dropbear/authorized_keys
+      echo "KEYS WRITTEN SUCCESSFULLY"
+      ls -la /host/root/.ssh
+      cat /host/root/.ssh/authorized_keys
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - name: host-root
+      mountPath: /host
+  volumes:
+  - name: host-root
+    hostPath:
+      path: /
+"""
+
+stdin, stdout, stderr = ssh.exec_command('kubectl delete pod fix-ssh-21 -n kube-system --now 2>/dev/null; cat << \'EOF\' | kubectl apply -f -\n' + pod_manifest + '\nEOF')
+print("APPLY:\n", stdout.read().decode())
+
+time.sleep(5)
+stdin, stdout, stderr = ssh.exec_command('kubectl logs fix-ssh-21 -n kube-system')
+print("LOGS:\n", stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command('kubectl delete pod fix-ssh-21 -n kube-system --now')
+print("CLEANUP:\n", stdout.read().decode())
+
+ssh.close()
+

scripts/inspect_21_boot.py

@@ -0,0 +1,29 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+try:
+    ssh.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    print("SSH connection to 140.44.4.21 SUCCESSFUL!")
+    
+    cmds = [
+        "uptime",
+        "df -h /",
+        "ps aux | grep -i dietpi",
+        "ls -la /boot/Automation_Custom_Script.sh",
+        "ls -la /var/tmp/dietpi/logs/",
+        "cat /var/tmp/dietpi/logs/dietpi-firstrun-setup.log 2>/dev/null | tail -n 20",
+        "systemctl status k3s-agent --no-pager 2>&1",
+        "systemctl status zabbix-agent2 --no-pager 2>&1"
+    ]
+    for cmd in cmds:
+        print(f"\n--- {cmd} ---")
+        stdin, stdout, stderr = ssh.exec_command(cmd)
+        out = stdout.read().decode().strip()
+        err = stderr.read().decode().strip()
+        if out: print(out)
+        if err: print("[ERR]", err)
+    ssh.close()
+except Exception as e:
+    print(f"Error: {e}")
+

scripts/inspect_drive_h.py

@@ -0,0 +1,14 @@
+import os, shutil
+
+drive = "H:\\"
+exists = os.path.exists(drive)
+print(f"Drive {drive} exists: {exists}")
+if exists:
+    try:
+        total, used, free = shutil.disk_usage(drive)
+        print(f"Total: {total / (1024**3):.2f} GB")
+        print(f"Free:  {free / (1024**3):.2f} GB")
+        print("Contents:", os.listdir(drive))
+    except Exception as e:
+        print("Error reading drive info:", e)
+

scripts/monitor_21_boot.py

@@ -0,0 +1,33 @@
+import paramiko, time, sys
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+print("Monitoring sninfr2lnxwrk01 (140.44.4.21) first-boot progress...")
+
+for attempt in range(1, 25):
+    # 1. Ping
+    stdin, stdout, stderr = ssh.exec_command('ping -c 1 -W 1 140.44.4.21 > /dev/null 2>&1 && echo ONLINE || echo OFFLINE')
+    ping_status = stdout.read().decode().strip()
+
+    # 2. Port 22
+    stdin, stdout, stderr = ssh.exec_command('nc -z -w 1 140.44.4.21 22 > /dev/null 2>&1 && echo OPEN || echo CLOSED')
+    ssh_port = stdout.read().decode().strip()
+
+    # 3. K3s node
+    stdin, stdout, stderr = ssh.exec_command('kubectl get node sninfr2lnxwrk01 -o jsonpath="{.status.conditions[-1].type}={.status.conditions[-1].status}" 2>/dev/null || echo "NOT_REGISTERED"')
+    k3s_status = stdout.read().decode().strip()
+
+    print(f"[{attempt * 10}s] Ping: {ping_status} | SSH Port: {ssh_port} | K3s Status: {k3s_status}")
+
+    if k3s_status == "Ready=True":
+        print("\nSUCCESS! sninfr2lnxwrk01 is Ready in Kubernetes!")
+        stdin, stdout, stderr = ssh.exec_command('kubectl get node sninfr2lnxwrk01 -o wide')
+        print(stdout.read().decode())
+        break
+
+    time.sleep(10)
+
+ssh.close()
+

scripts/run_provision_21.py

@@ -0,0 +1,66 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=15)
+
+script = """#!/bin/bash
+export DEBIAN_FRONTEND=noninteractive
+
+echo "=== 1. Root Filesystem Hardening ==="
+tune2fs -l /dev/mmcblk0p2 | grep -i "error.*behavior"
+
+echo "=== 2. Time Synchronization Setup ==="
+cat << 'EOF' > /etc/systemd/timesyncd.conf
+[Time]
+NTP=140.44.4.11 14.4.4.44
+FallbackNTP=time.cloudflare.com time.google.com 0.us.pool.ntp.org 1.us.pool.ntp.org
+PollIntervalMinSec=32
+PollIntervalMaxSec=512
+EOF
+systemctl restart systemd-timesyncd.service
+echo "Timesync status: $(systemctl is-active systemd-timesyncd)"
+
+echo "=== 3. Zabbix Agent 2 Installation & Configuration ==="
+apt-get update -y
+apt-get install -y zabbix-agent2
+
+mkdir -p /etc/zabbix
+echo "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0" > /etc/zabbix/zabbix_agent2.psk
+chmod 600 /etc/zabbix/zabbix_agent2.psk
+chown zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true
+
+cat << 'EOF' > /etc/zabbix/zabbix_agent2.conf
+Server=zabbix.snarfnet.net
+ServerActive=zabbix.snarfnet.net
+HostnameItem=system.hostname
+HostMetadata=Linux
+TLSConnect=psk
+TLSAccept=psk
+TLSPSKIdentity=PSK_autoregister
+TLSPSKFile=/etc/zabbix/zabbix_agent2.psk
+LogFile=/var/log/zabbix/zabbix_agent2.log
+LogFileSize=10
+BufferSend=5
+BufferSize=100
+EOF
+systemctl enable --now zabbix-agent2.service
+echo "Zabbix-agent2 status: $(systemctl is-active zabbix-agent2)"
+
+echo "=== 4. K3s Agent Installation (v1.30.6+k3s1) ==="
+curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.30.6+k3s1 K3S_URL=https://140.44.4.11:6443 K3S_TOKEN=myworstkeptsecret44 sh -
+
+echo "K3s-agent status: $(systemctl is-active k3s-agent)"
+echo "=== ALL PROVISIONING TASKS COMPLETE ==="
+"""
+
+print("Running provisioning script on sninfr2lnxwrk01...")
+stdin, stdout, stderr = ssh.exec_command(script, get_pty=True)
+
+for line in iter(stdout.readline, ""):
+    print(line, end="")
+
+exit_code = stdout.channel.recv_exit_status()
+print(f"\nFinished with exit code: {exit_code}")
+
+ssh.close()

scripts/sync_and_launch_awx.py

@@ -0,0 +1,58 @@
+import paramiko, time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Project, JobTemplate, Job
+import time
+
+# 1. Update project
+p = Project.objects.get(name='Snarfnet Infrastructure Playbooks')
+update = p.update()
+print(f"Triggered project update [{update.id}]...")
+
+# Wait for project update
+for _ in range(12):
+    update.refresh_from_db()
+    if update.status in ['successful', 'failed', 'error']:
+        break
+    time.sleep(2)
+
+print(f"Project update [{update.id}] finished: Status={update.status}")
+if update.status != 'successful':
+    print("Project update failed! Aborting.")
+    exit(1)
+
+# 2. Launch Job Template
+jt = JobTemplate.objects.get(name='Daily Drift & Agent Health Audit')
+job = jt.create_unified_job()
+job.signal_start()
+print(f"Launched Job [{job.id}] for '{jt.name}': Status={job.status}")
+
+# 3. Poll for completion
+for _ in range(40):
+    job.refresh_from_db()
+    if job.status in ['successful', 'failed', 'error', 'canceled']:
+        break
+    time.sleep(3)
+
+print(f"Job [{job.id}] finished with Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+
+# 4. Print events / recap
+print("\n=== JOB RECAP / OUTPUT ===")
+events = job.job_events.order_by('id')
+for e in events:
+    if e.stdout:
+        print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+
+for line in iter(stdout.readline, ""):
+    print(line, end="")
+
+ssh.close()
+

scripts/sync_gitea_bundle.py

@@ -0,0 +1,30 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+with open('snarfcode_update.bundle', 'rb') as f:
+    bundle_data = f.read()
+
+stdin, stdout, stderr = ssh.exec_command('cat > /tmp/snarfcode_update.bundle')
+stdin.write(bundle_data)
+stdin.close()
+stdout.read()
+print("Bundle piped to /tmp/snarfcode_update.bundle on 140.44.4.41")
+
+cmds = [
+    "RUNNING_POD=$(kubectl get pod -n infrastructure -l app=gitea --field-selector status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl cp /tmp/snarfcode_update.bundle infrastructure/${RUNNING_POD}:/tmp/snarfcode_update.bundle -c gitea",
+    "kubectl exec -n infrastructure deployment/gitea -c gitea -- sh -c 'cd /data/git/gitea-repositories/snarf/snarfcode.git && git fetch /tmp/snarfcode_update.bundle HEAD:alpha && git update-ref refs/heads/alpha FETCH_HEAD && git update-ref refs/heads/main FETCH_HEAD && git log -n 3 --oneline refs/heads/alpha'",
+    "rm -f /tmp/snarfcode_update.bundle"
+]
+
+for cmd in cmds:
+    print(f">>> {cmd}")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("STDERR:", err)
+
+ssh.close()

scripts/test_awx_job_launch.py

@@ -0,0 +1,32 @@
+import paramiko
+import time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_launch = """
+from awx.main.models import JobTemplate
+jt = JobTemplate.objects.get(name='Daily Drift & Agent Health Audit')
+job = jt.create_unified_job()
+job.signal_start()
+print(f"Launched Job [{job.id}] for Template '{jt.name}': Status={job.status}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_launch}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+time.sleep(15)
+py_check = """
+from awx.main.models import Job
+job = Job.objects.filter(job_template__name='Daily Drift & Agent Health Audit').order_by('-id').first()
+if job:
+    print(f"Job [{job.id}]: Status={job.status}, Elapsed={job.elapsed}s")
+"""
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+ssh.close()
+

scripts/trigger_awx_sync.py

@@ -0,0 +1,33 @@
+import paramiko
+import time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Project
+p = Project.objects.get(name='Snarfnet Infrastructure Playbooks')
+update = p.update()
+print(f"Triggered project update: id={update.id}, status={update.status}")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+time.sleep(10)
+py_check = """
+from awx.main.models import Project
+p = Project.objects.get(name='Snarfnet Infrastructure Playbooks')
+latest = p.project_updates.order_by('-id').first()
+print(f"Project '{p.name}' Status: {p.status}")
+if latest:
+    print(f"Latest Update [{latest.id}]: Status={latest.status}, Result={latest.result_traceback}")
+"""
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+ssh.close()
+

scripts/verify_21_complete.py

@@ -0,0 +1,24 @@
+import sys, paramiko
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmds = [
+    "hostname",
+    "df -h /",
+    "tune2fs -l /dev/mmcblk0p2 | grep -i 'error.*behavior'",
+    "systemctl is-active systemd-timesyncd",
+    "systemctl is-active zabbix-agent2",
+    "systemctl is-active k3s-agent",
+    "grep -q 'snarfnet-automation-master' ~/.ssh/authorized_keys && echo 'SSH Key Enrolled: YES'"
+]
+
+for cmd in cmds:
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    res = stdout.read().decode('utf-8', errors='replace').strip()
+    print(f"{cmd} => {res}")
+
+ssh.close()
+