feat(infra): Phase 3 & 4 completion, deploy AWX playbooks & Zabbix monitoring

ansible/playbooks/daily_drift_audit.yml

@@ -0,0 +1,26 @@
+---
+- name: Daily Infrastructure Health & Configuration Drift Audit
+  hosts: all:!synology_nas
+  gather_facts: false
+  tasks:
+    - name: Audit system time sync status
+      ansible.builtin.command: timedatectl show -p NTPSynchronized --value
+      register: time_sync
+      changed_when: false
+      failed_when: time_sync.stdout != "yes"
+
+    - name: Audit Zabbix Agent 2 service status
+      ansible.builtin.command: systemctl is-active zabbix-agent2
+      register: zabbix_status
+      changed_when: false
+      failed_when: zabbix_status.stdout != "active"
+
+    - name: Audit unified SSH key enrollment in authorized_keys
+      ansible.builtin.shell: grep -q "snarfnet-automation-master@snarfnet.net" ~/.ssh/authorized_keys
+      register: ssh_key_check
+      changed_when: false
+      failed_when: ssh_key_check.rc != 0
+
+    - name: Display host audit status
+      ansible.builtin.debug:
+        msg: "{{ inventory_hostname }} ({{ ansible_host }}) [OK]: TimeSync={{ time_sync.stdout }}, Zabbix={{ zabbix_status.stdout }}, SSHKey=Verified"

ansible/playbooks/k3s_aerodrome_precheck.yml

@@ -0,0 +1,29 @@
+---
+- name: Aerodrome K3s Control Plane Pre-Upgrade Health & etcd Snapshot
+  hosts: k3s_aerodrome_masters
+  gather_facts: false
+  tasks:
+    - name: Verify K3s systemd service is active
+      ansible.builtin.command: systemctl is-active k3s
+      register: k3s_service
+      changed_when: false
+      failed_when: k3s_service.stdout != "active"
+
+    - name: Query current node K3s version
+      ansible.builtin.command: k3s --version
+      register: k3s_ver
+      changed_when: false
+
+    - name: Display current K3s version
+      ansible.builtin.debug:
+        msg: "{{ inventory_hostname }} current version: {{ k3s_ver.stdout_lines[0] }}"
+
+    - name: Save on-demand etcd snapshot on primary master (.11)
+      ansible.builtin.command: k3s etcd-snapshot save --name "pre-upgrade-snapshot"
+      when: inventory_hostname == "sninfr2lnxctl01"
+      register: etcd_snap
+
+    - name: Report etcd snapshot result
+      ansible.builtin.debug:
+        msg: "{{ etcd_snap.stdout_lines }}"
+      when: inventory_hostname == "sninfr2lnxctl01"

ansible/playbooks/weekly_os_patching.yml

@@ -0,0 +1,46 @@
+---
+- name: Weekly OS & Security Patching with Rolling Safety Gates
+  hosts: all:!synology_nas
+  serial: 1
+  gather_facts: true
+
+  tasks:
+    - name: Update apt cache
+      ansible.builtin.apt:
+        update_cache: true
+        cache_valid_time: 3600
+      when: ansible_os_family == "Debian"
+
+    - name: Perform dist-upgrade
+      ansible.builtin.apt:
+        upgrade: dist
+        autoremove: true
+        autoclean: true
+      when: ansible_os_family == "Debian"
+
+    - name: Check if system reboot is required
+      ansible.builtin.stat:
+        path: /var/run/reboot-required
+      register: reboot_required_file
+
+    - name: Rolling Reboot with Connection Verification
+      when: reboot_required_file.stat.exists
+      block:
+        - name: Reboot host gracefully
+          ansible.builtin.reboot:
+            msg: "Automated rolling maintenance reboot via AWX"
+            reboot_timeout: 300
+            connect_timeout: 30
+            test_command: uptime
+
+        - name: Verify systemd-timesyncd is active after reboot
+          ansible.builtin.systemd:
+            name: systemd-timesyncd
+            state: started
+          ignore_errors: true
+
+        - name: Verify zabbix-agent2 is active after reboot
+          ansible.builtin.systemd:
+            name: zabbix-agent2
+            state: started
+          ignore_errors: true