fix(patching): add debconf non-interactive pre-seeding, cordon/drain, and storage settling to weekly OS patching

ansible/playbooks/weekly_os_patching.yml

@@ -5,10 +5,46 @@
   gather_facts: true
 
   tasks:
+    - name: Pre-seed debconf for non-interactive kernel and package upgrades
+      ansible.builtin.shell: |
+        echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
+        echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
+      when: ansible_os_family == "Debian"
+      changed_when: false
+
+    - name: Ensure critical storage prerequisites are installed
+      ansible.builtin.apt:
+        name:
+          - open-iscsi
+          - nfs-common
+        state: present
+        update_cache: false
+      when: ansible_os_family == "Debian" and node_role is defined and node_role in ['master', 'worker']
+
+    - name: Cordon Kubernetes node if applicable
+      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      changed_when: true
+
+    - name: Drain Kubernetes node if applicable
+      ansible.builtin.command: >
+        kubectl drain {{ inventory_hostname }}
+        --ignore-daemonsets
+        --delete-emptydir-data
+        --force
+        --timeout=180s
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      failed_when: false
+      changed_when: true
+
     - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 3600
+      environment:
+        DEBIAN_FRONTEND: noninteractive
       when: ansible_os_family == "Debian"
 
     - name: Perform dist-upgrade
@@ -16,6 +52,9 @@
         upgrade: dist
         autoremove: true
         autoclean: true
+        dpkg_options: 'force-confdef,force-confold'
+      environment:
+        DEBIAN_FRONTEND: noninteractive
       when: ansible_os_family == "Debian"
 
     - name: Check if system reboot is required
@@ -28,16 +67,18 @@
       block:
         - name: Reboot host gracefully
           ansible.builtin.reboot:
-            msg: "Automated rolling maintenance reboot via AWX"
+            msg: "Automated rolling maintenance reboot via Snarfnet Automation"
             reboot_timeout: 300
             connect_timeout: 30
             test_command: uptime
 
-        - name: Verify systemd-timesyncd is active after reboot
-          ansible.builtin.systemd:
-            name: systemd-timesyncd
-            state: started
-          ignore_errors: true
+        - name: Verify systemd-timesyncd or chrony is active after reboot
+          ansible.builtin.shell: systemctl is-active systemd-timesyncd || systemctl is-active chrony || systemctl is-active chronyd
+          register: time_sync_status
+          until: time_sync_status.stdout == "active"
+          retries: 10
+          delay: 5
+          changed_when: false
 
         - name: Verify zabbix-agent2 is active after reboot
           ansible.builtin.systemd:
@@ -45,3 +86,25 @@
             state: started
           ignore_errors: true
 
+    - name: Uncordon Kubernetes node if applicable
+      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      changed_when: true
+
+    - name: Wait for Kubernetes node to report Ready
+      ansible.builtin.command: >
+        kubectl get node {{ inventory_hostname }}
+        -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      register: node_ready
+      until: node_ready.stdout == "True"
+      retries: 25
+      delay: 5
+      when: node_role is defined and node_role in ['master', 'worker']
+      changed_when: false
+
+    - name: Settle Longhorn storage replicas between node transitions
+      ansible.builtin.pause:
+        seconds: 15
+      when: node_role is defined and node_role in ['master', 'worker']