feat(infra): Phase 3 & 4 completion, deploy AWX playbooks & Zabbix monitoring

ansible/inventories/snarfnet.yml

@@ -0,0 +1,83 @@
+all:
+  children:
+    k3s_aerodrome_masters:
+      vars:
+        ansible_user: root
+        cluster: aerodrome
+        node_role: master
+      hosts:
+        sninfr2lnxctl01: { ansible_host: 140.44.4.11 }
+        sninfr2lnxctl02: { ansible_host: 140.44.4.12 }
+        sninfr2lnxctl03: { ansible_host: 140.44.4.13 }
+
+    k3s_aerodrome_workers:
+      vars:
+        ansible_user: root
+        cluster: aerodrome
+        node_role: worker
+      hosts:
+        sninfr2lnxwrk01: { ansible_host: 140.44.4.21 }
+        sninfr2lnxwrk02: { ansible_host: 140.44.4.22 }
+        sninfr2lnxwrk03: { ansible_host: 140.44.4.23 }
+        sninfr2lnxwrk04: { ansible_host: 140.44.4.24 }
+        sninfr2lnxwrk05: { ansible_host: 140.44.4.25 }
+        sninfr2lnxwrk06: { ansible_host: 140.44.4.26 }
+        sninfr2lnxwrk07: { ansible_host: 140.44.4.27 }
+
+    k3s_spaceport_masters:
+      vars:
+        ansible_user: root
+        cluster: spaceport
+        node_role: master
+      hosts:
+        sninfr0lnxctl01: { ansible_host: 140.44.4.41 }
+        sninfr0lnxctl02: { ansible_host: 140.44.4.42 }
+        sninfr0lnxctl03: { ansible_host: 140.44.4.43 }
+        sninfr0lnxctl04: { ansible_host: 140.44.4.44 }
+
+    k3s_spaceport_workers:
+      vars:
+        ansible_user: root
+        cluster: spaceport
+        node_role: worker
+      hosts:
+        sninfr0lnxwrk01: { ansible_host: 140.44.4.51 }
+        sninfr0lnxwrk02: { ansible_host: 140.44.4.52 }
+        sninfr0lnxwrk03: { ansible_host: 140.44.4.53 }
+        sninfr0lnxwrk04: { ansible_host: 140.44.4.54 }
+        sninfr0lnxwrk05: { ansible_host: 140.44.4.55 }
+        sninfr0lnxwrk06: { ansible_host: 140.44.4.56 }
+
+    swarm_shipyard:
+      vars:
+        ansible_user: snarf
+        ansible_become: true
+        cluster: shipyard
+        node_role: swarm
+      hosts:
+        snshpd4lnxswm01: { ansible_host: 140.44.4.71 }
+        snshpd4lnxswm02: { ansible_host: 140.44.4.72 }
+        snshpd4lnxswm03: { ansible_host: 140.44.4.73 }
+        snshpd4lnxswm04: { ansible_host: 140.44.4.74 }
+        snshpd4lnxswm05: { ansible_host: 140.44.4.75 }
+
+    ai_pi_cluster:
+      vars:
+        ansible_user: snarf
+        ansible_become: true
+      hosts:
+        sninfr4lnxaih01: { ansible_host: 140.44.4.7 }
+        sninfr4lnxaih02: { ansible_host: 140.44.4.8 }
+
+    smart_home_nodes:
+      vars:
+        ansible_user: snarf
+        ansible_become: true
+      hosts:
+        homeassistant: { ansible_host: 14.4.4.20 }
+        nutserver: { ansible_host: 14.4.4.44 }
+
+    synology_nas:
+      hosts:
+        sn_nas: { ansible_host: 14.10.10.71 }
+        snarfnas: { ansible_host: 14.10.10.72 }

ansible/playbooks/daily_drift_audit.yml

@@ -0,0 +1,26 @@
+---
+- name: Daily Infrastructure Health & Configuration Drift Audit
+  hosts: all:!synology_nas
+  gather_facts: false
+  tasks:
+    - name: Audit system time sync status
+      ansible.builtin.command: timedatectl show -p NTPSynchronized --value
+      register: time_sync
+      changed_when: false
+      failed_when: time_sync.stdout != "yes"
+
+    - name: Audit Zabbix Agent 2 service status
+      ansible.builtin.command: systemctl is-active zabbix-agent2
+      register: zabbix_status
+      changed_when: false
+      failed_when: zabbix_status.stdout != "active"
+
+    - name: Audit unified SSH key enrollment in authorized_keys
+      ansible.builtin.shell: grep -q "snarfnet-automation-master@snarfnet.net" ~/.ssh/authorized_keys
+      register: ssh_key_check
+      changed_when: false
+      failed_when: ssh_key_check.rc != 0
+
+    - name: Display host audit status
+      ansible.builtin.debug:
+        msg: "{{ inventory_hostname }} ({{ ansible_host }}) [OK]: TimeSync={{ time_sync.stdout }}, Zabbix={{ zabbix_status.stdout }}, SSHKey=Verified"

ansible/playbooks/k3s_aerodrome_precheck.yml

@@ -0,0 +1,29 @@
+---
+- name: Aerodrome K3s Control Plane Pre-Upgrade Health & etcd Snapshot
+  hosts: k3s_aerodrome_masters
+  gather_facts: false
+  tasks:
+    - name: Verify K3s systemd service is active
+      ansible.builtin.command: systemctl is-active k3s
+      register: k3s_service
+      changed_when: false
+      failed_when: k3s_service.stdout != "active"
+
+    - name: Query current node K3s version
+      ansible.builtin.command: k3s --version
+      register: k3s_ver
+      changed_when: false
+
+    - name: Display current K3s version
+      ansible.builtin.debug:
+        msg: "{{ inventory_hostname }} current version: {{ k3s_ver.stdout_lines[0] }}"
+
+    - name: Save on-demand etcd snapshot on primary master (.11)
+      ansible.builtin.command: k3s etcd-snapshot save --name "pre-upgrade-snapshot"
+      when: inventory_hostname == "sninfr2lnxctl01"
+      register: etcd_snap
+
+    - name: Report etcd snapshot result
+      ansible.builtin.debug:
+        msg: "{{ etcd_snap.stdout_lines }}"
+      when: inventory_hostname == "sninfr2lnxctl01"

ansible/playbooks/weekly_os_patching.yml

@@ -0,0 +1,46 @@
+---
+- name: Weekly OS & Security Patching with Rolling Safety Gates
+  hosts: all:!synology_nas
+  serial: 1
+  gather_facts: true
+
+  tasks:
+    - name: Update apt cache
+      ansible.builtin.apt:
+        update_cache: true
+        cache_valid_time: 3600
+      when: ansible_os_family == "Debian"
+
+    - name: Perform dist-upgrade
+      ansible.builtin.apt:
+        upgrade: dist
+        autoremove: true
+        autoclean: true
+      when: ansible_os_family == "Debian"
+
+    - name: Check if system reboot is required
+      ansible.builtin.stat:
+        path: /var/run/reboot-required
+      register: reboot_required_file
+
+    - name: Rolling Reboot with Connection Verification
+      when: reboot_required_file.stat.exists
+      block:
+        - name: Reboot host gracefully
+          ansible.builtin.reboot:
+            msg: "Automated rolling maintenance reboot via AWX"
+            reboot_timeout: 300
+            connect_timeout: 30
+            test_command: uptime
+
+        - name: Verify systemd-timesyncd is active after reboot
+          ansible.builtin.systemd:
+            name: systemd-timesyncd
+            state: started
+          ignore_errors: true
+
+        - name: Verify zabbix-agent2 is active after reboot
+          ansible.builtin.systemd:
+            name: zabbix-agent2
+            state: started
+          ignore_errors: true