diff --git a/ansible/playbooks/weekly_os_patching.yml b/ansible/playbooks/weekly_os_patching.yml index e3d8f7f..4107cfa 100644 --- a/ansible/playbooks/weekly_os_patching.yml +++ b/ansible/playbooks/weekly_os_patching.yml @@ -5,10 +5,46 @@ gather_facts: true tasks: + - name: Pre-seed debconf for non-interactive kernel and package upgrades + ansible.builtin.shell: | + echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections + echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections + when: ansible_os_family == "Debian" + changed_when: false + + - name: Ensure critical storage prerequisites are installed + ansible.builtin.apt: + name: + - open-iscsi + - nfs-common + state: present + update_cache: false + when: ansible_os_family == "Debian" and node_role is defined and node_role in ['master', 'worker'] + + - name: Cordon Kubernetes node if applicable + ansible.builtin.command: kubectl cordon {{ inventory_hostname }} + delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}" + when: node_role is defined and node_role == 'worker' + changed_when: true + + - name: Drain Kubernetes node if applicable + ansible.builtin.command: > + kubectl drain {{ inventory_hostname }} + --ignore-daemonsets + --delete-emptydir-data + --force + --timeout=180s + delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}" + when: node_role is defined and node_role == 'worker' + failed_when: false + changed_when: true + - name: Update apt cache ansible.builtin.apt: update_cache: true cache_valid_time: 3600 + environment: + DEBIAN_FRONTEND: noninteractive when: ansible_os_family == "Debian" - name: Perform dist-upgrade @@ -16,6 +52,9 @@ upgrade: dist autoremove: true autoclean: true + dpkg_options: 'force-confdef,force-confold' + environment: + DEBIAN_FRONTEND: noninteractive when: ansible_os_family == "Debian" - name: Check if system reboot is required @@ -28,16 +67,18 @@ block: - name: Reboot host gracefully ansible.builtin.reboot: - msg: "Automated rolling maintenance reboot via AWX" + msg: "Automated rolling maintenance reboot via Snarfnet Automation" reboot_timeout: 300 connect_timeout: 30 test_command: uptime - - name: Verify systemd-timesyncd is active after reboot - ansible.builtin.systemd: - name: systemd-timesyncd - state: started - ignore_errors: true + - name: Verify systemd-timesyncd or chrony is active after reboot + ansible.builtin.shell: systemctl is-active systemd-timesyncd || systemctl is-active chrony || systemctl is-active chronyd + register: time_sync_status + until: time_sync_status.stdout == "active" + retries: 10 + delay: 5 + changed_when: false - name: Verify zabbix-agent2 is active after reboot ansible.builtin.systemd: @@ -45,3 +86,25 @@ state: started ignore_errors: true + - name: Uncordon Kubernetes node if applicable + ansible.builtin.command: kubectl uncordon {{ inventory_hostname }} + delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}" + when: node_role is defined and node_role == 'worker' + changed_when: true + + - name: Wait for Kubernetes node to report Ready + ansible.builtin.command: > + kubectl get node {{ inventory_hostname }} + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}" + register: node_ready + until: node_ready.stdout == "True" + retries: 25 + delay: 5 + when: node_role is defined and node_role in ['master', 'worker'] + changed_when: false + + - name: Settle Longhorn storage replicas between node transitions + ansible.builtin.pause: + seconds: 15 + when: node_role is defined and node_role in ['master', 'worker']