SnarfCode/ansible/playbooks/upgrade_k3s_aerodrome.yml

---
- name: Aerodrome K3s Control Plane Sequential Upgrade
  hosts: k3s_aerodrome_masters
  gather_facts: false
  serial: 1
  vars:
    target_k3s_version: "v1.31.4+k3s1"

  tasks:
    - name: Query pre-upgrade K3s version
      ansible.builtin.command: k3s --version
      register: pre_k3s_ver
      changed_when: false

    - name: Display pre-upgrade version
      ansible.builtin.debug:
        msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"

    - name: Download target K3s release binary
      ansible.builtin.shell: |
        set -e
        ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
        curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
        chmod +x /usr/local/bin/k3s.new
        mv /usr/local/bin/k3s.new /usr/local/bin/k3s
      when: target_k3s_version not in pre_k3s_ver.stdout
      register: k3s_download

    - name: Restart K3s systemd service
      ansible.builtin.systemd:
        name: k3s
        state: restarted
      when: target_k3s_version not in pre_k3s_ver.stdout

    - name: Wait for K3s systemd unit to report active
      ansible.builtin.command: systemctl is-active k3s
      register: k3s_active
      until: k3s_active.stdout == "active"
      retries: 25
      delay: 4
      changed_when: false
      when: target_k3s_version not in pre_k3s_ver.stdout

    - name: Wait for upgraded master to report Ready in Kubernetes
      ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
      delegate_to: sninfr2lnxctl01
      register: node_ready
      until: node_ready.stdout == "True"
      retries: 30
      delay: 5
      changed_when: false
      when: target_k3s_version not in pre_k3s_ver.stdout

    - name: Query post-upgrade K3s version
      ansible.builtin.command: k3s --version
      register: post_k3s_ver
      changed_when: false

    - name: Verify post-upgrade version matches target
      ansible.builtin.assert:
        that:
          - target_k3s_version in post_k3s_ver.stdout
        fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
        success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"

    - name: Settle etcd cluster consensus
      ansible.builtin.pause:
        seconds: 15
      when: target_k3s_version not in pre_k3s_ver.stdout

- name: Aerodrome K3s Worker Nodes Sequential Upgrade
  hosts: k3s_aerodrome_workers
  gather_facts: false
  serial: 1
  vars:
    target_k3s_version: "v1.31.4+k3s1"

  tasks:
    - name: Query pre-upgrade worker K3s version
      ansible.builtin.command: k3s --version
      register: pre_k3s_ver
      changed_when: false

    - name: Display pre-upgrade worker version
      ansible.builtin.debug:
        msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"

    - name: Cordon worker node
      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
      delegate_to: sninfr2lnxctl01
      when: target_k3s_version not in pre_k3s_ver.stdout
      changed_when: true

    - name: Drain worker node
      ansible.builtin.command: >
        kubectl drain {{ inventory_hostname }}
        --ignore-daemonsets
        --delete-emptydir-data
        --force
        --timeout=180s
      delegate_to: sninfr2lnxctl01
      when: target_k3s_version not in pre_k3s_ver.stdout
      failed_when: false
      changed_when: true

    - name: Download target K3s release binary on worker
      ansible.builtin.shell: |
        set -e
        ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
        curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
        chmod +x /usr/local/bin/k3s.new
        mv /usr/local/bin/k3s.new /usr/local/bin/k3s
      when: target_k3s_version not in pre_k3s_ver.stdout
      register: k3s_download_worker

    - name: Restart K3s Agent systemd service
      ansible.builtin.systemd:
        name: k3s-agent
        state: restarted
      when: target_k3s_version not in pre_k3s_ver.stdout

    - name: Wait for K3s Agent systemd unit to report active
      ansible.builtin.command: systemctl is-active k3s-agent
      register: agent_active
      until: agent_active.stdout == "active"
      retries: 25
      delay: 4
      changed_when: false
      when: target_k3s_version not in pre_k3s_ver.stdout

    - name: Uncordon worker node
      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
      delegate_to: sninfr2lnxctl01
      when: target_k3s_version not in pre_k3s_ver.stdout
      changed_when: true

    - name: Wait for upgraded worker to report Ready in Kubernetes
      ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
      delegate_to: sninfr2lnxctl01
      register: node_ready
      until: node_ready.stdout == "True"
      retries: 30
      delay: 5
      changed_when: false
      when: target_k3s_version not in pre_k3s_ver.stdout

    - name: Query post-upgrade worker K3s version
      ansible.builtin.command: k3s --version
      register: post_k3s_ver
      changed_when: false

    - name: Verify post-upgrade worker version matches target
      ansible.builtin.assert:
        that:
          - target_k3s_version in post_k3s_ver.stdout
        fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
        success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"

    - name: Settle Longhorn storage replicas between node transitions
      ansible.builtin.pause:
        seconds: 15
      when: target_k3s_version not in pre_k3s_ver.stdout