feat(k3s): upgrade Aerodrome control plane and workers to v1.31.4+k3s1 with rolling drain and safety gates

ansible/playbooks/upgrade_k3s_aerodrome.yml

@@ -1,10 +1,10 @@
 ---
-- name: Aerodrome K3s Control Plane Sequential Minor Version Upgrade
+- name: Aerodrome K3s Control Plane Sequential Upgrade
   hosts: k3s_aerodrome_masters
   gather_facts: false
   serial: 1
   vars:
-    target_k3s_version: "v1.27.16+k3s1"
+    target_k3s_version: "v1.31.4+k3s1"
 
   tasks:
     - name: Query pre-upgrade K3s version
@@ -16,7 +16,7 @@
       ansible.builtin.debug:
         msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
 
-    - name: Download target K3s release binary and replace executable
+    - name: Download target K3s release binary
       ansible.builtin.shell: |
         set -e
         ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
@@ -36,12 +36,12 @@
       ansible.builtin.command: systemctl is-active k3s
       register: k3s_active
       until: k3s_active.stdout == "active"
-      retries: 20
+      retries: 25
       delay: 4
       changed_when: false
       when: target_k3s_version not in pre_k3s_ver.stdout
 
-    - name: Wait for upgraded node to report Ready in Kubernetes
+    - name: Wait for upgraded master to report Ready in Kubernetes
       ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
       delegate_to: sninfr2lnxctl01
       register: node_ready
@@ -63,7 +63,100 @@
         fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
         success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
 
-    - name: Settle etcd cluster state
+    - name: Settle etcd cluster consensus
       ansible.builtin.pause:
-        seconds: 10
+        seconds: 15
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+- name: Aerodrome K3s Worker Nodes Sequential Upgrade
+  hosts: k3s_aerodrome_workers
+  gather_facts: false
+  serial: 1
+  vars:
+    target_k3s_version: "v1.31.4+k3s1"
+
+  tasks:
+    - name: Query pre-upgrade worker K3s version
+      ansible.builtin.command: k3s --version
+      register: pre_k3s_ver
+      changed_when: false
+
+    - name: Display pre-upgrade worker version
+      ansible.builtin.debug:
+        msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
+
+    - name: Cordon worker node
+      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      changed_when: true
+
+    - name: Drain worker node
+      ansible.builtin.command: >
+        kubectl drain {{ inventory_hostname }}
+        --ignore-daemonsets
+        --delete-emptydir-data
+        --force
+        --timeout=180s
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      failed_when: false
+      changed_when: true
+
+    - name: Download target K3s release binary on worker
+      ansible.builtin.shell: |
+        set -e
+        ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
+        curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
+        chmod +x /usr/local/bin/k3s.new
+        mv /usr/local/bin/k3s.new /usr/local/bin/k3s
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      register: k3s_download_worker
+
+    - name: Restart K3s Agent systemd service
+      ansible.builtin.systemd:
+        name: k3s-agent
+        state: restarted
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Wait for K3s Agent systemd unit to report active
+      ansible.builtin.command: systemctl is-active k3s-agent
+      register: agent_active
+      until: agent_active.stdout == "active"
+      retries: 25
+      delay: 4
+      changed_when: false
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Uncordon worker node
+      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      changed_when: true
+
+    - name: Wait for upgraded worker to report Ready in Kubernetes
+      ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      delegate_to: sninfr2lnxctl01
+      register: node_ready
+      until: node_ready.stdout == "True"
+      retries: 30
+      delay: 5
+      changed_when: false
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Query post-upgrade worker K3s version
+      ansible.builtin.command: k3s --version
+      register: post_k3s_ver
+      changed_when: false
+
+    - name: Verify post-upgrade worker version matches target
+      ansible.builtin.assert:
+        that:
+          - target_k3s_version in post_k3s_ver.stdout
+        fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
+        success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
+
+    - name: Settle Longhorn storage replicas between node transitions
+      ansible.builtin.pause:
+        seconds: 15
       when: target_k3s_version not in pre_k3s_ver.stdout