feat(k3s): upgrade Aerodrome control plane and workers to v1.31.4+k3s1 with rolling drain and safety gates

ansible/playbooks/upgrade_k3s_aerodrome.yml

@@ -1,10 +1,10 @@
 ---
-- name: Aerodrome K3s Control Plane Sequential Minor Version Upgrade
+- name: Aerodrome K3s Control Plane Sequential Upgrade
   hosts: k3s_aerodrome_masters
   gather_facts: false
   serial: 1
   vars:
-    target_k3s_version: "v1.27.16+k3s1"
+    target_k3s_version: "v1.31.4+k3s1"
 
   tasks:
     - name: Query pre-upgrade K3s version
@@ -16,7 +16,7 @@
       ansible.builtin.debug:
         msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
 
-    - name: Download target K3s release binary and replace executable
+    - name: Download target K3s release binary
       ansible.builtin.shell: |
         set -e
         ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
@@ -36,12 +36,12 @@
       ansible.builtin.command: systemctl is-active k3s
       register: k3s_active
       until: k3s_active.stdout == "active"
-      retries: 20
+      retries: 25
       delay: 4
       changed_when: false
       when: target_k3s_version not in pre_k3s_ver.stdout
 
-    - name: Wait for upgraded node to report Ready in Kubernetes
+    - name: Wait for upgraded master to report Ready in Kubernetes
       ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
       delegate_to: sninfr2lnxctl01
       register: node_ready
@@ -63,7 +63,100 @@
         fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
         success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
 
-    - name: Settle etcd cluster state
+    - name: Settle etcd cluster consensus
       ansible.builtin.pause:
-        seconds: 10
+        seconds: 15
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+- name: Aerodrome K3s Worker Nodes Sequential Upgrade
+  hosts: k3s_aerodrome_workers
+  gather_facts: false
+  serial: 1
+  vars:
+    target_k3s_version: "v1.31.4+k3s1"
+
+  tasks:
+    - name: Query pre-upgrade worker K3s version
+      ansible.builtin.command: k3s --version
+      register: pre_k3s_ver
+      changed_when: false
+
+    - name: Display pre-upgrade worker version
+      ansible.builtin.debug:
+        msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
+
+    - name: Cordon worker node
+      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      changed_when: true
+
+    - name: Drain worker node
+      ansible.builtin.command: >
+        kubectl drain {{ inventory_hostname }}
+        --ignore-daemonsets
+        --delete-emptydir-data
+        --force
+        --timeout=180s
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      failed_when: false
+      changed_when: true
+
+    - name: Download target K3s release binary on worker
+      ansible.builtin.shell: |
+        set -e
+        ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
+        curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
+        chmod +x /usr/local/bin/k3s.new
+        mv /usr/local/bin/k3s.new /usr/local/bin/k3s
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      register: k3s_download_worker
+
+    - name: Restart K3s Agent systemd service
+      ansible.builtin.systemd:
+        name: k3s-agent
+        state: restarted
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Wait for K3s Agent systemd unit to report active
+      ansible.builtin.command: systemctl is-active k3s-agent
+      register: agent_active
+      until: agent_active.stdout == "active"
+      retries: 25
+      delay: 4
+      changed_when: false
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Uncordon worker node
+      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      changed_when: true
+
+    - name: Wait for upgraded worker to report Ready in Kubernetes
+      ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      delegate_to: sninfr2lnxctl01
+      register: node_ready
+      until: node_ready.stdout == "True"
+      retries: 30
+      delay: 5
+      changed_when: false
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Query post-upgrade worker K3s version
+      ansible.builtin.command: k3s --version
+      register: post_k3s_ver
+      changed_when: false
+
+    - name: Verify post-upgrade worker version matches target
+      ansible.builtin.assert:
+        that:
+          - target_k3s_version in post_k3s_ver.stdout
+        fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
+        success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
+
+    - name: Settle Longhorn storage replicas between node transitions
+      ansible.builtin.pause:
+        seconds: 15
       when: target_k3s_version not in pre_k3s_ver.stdout

ansible/playbooks/weekly_os_patching.yml

@@ -5,10 +5,46 @@
   gather_facts: true
 
   tasks:
+    - name: Pre-seed debconf for non-interactive kernel and package upgrades
+      ansible.builtin.shell: |
+        echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
+        echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
+      when: ansible_os_family == "Debian"
+      changed_when: false
+
+    - name: Ensure critical storage prerequisites are installed
+      ansible.builtin.apt:
+        name:
+          - open-iscsi
+          - nfs-common
+        state: present
+        update_cache: false
+      when: ansible_os_family == "Debian" and node_role is defined and node_role in ['master', 'worker']
+
+    - name: Cordon Kubernetes node if applicable
+      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      changed_when: true
+
+    - name: Drain Kubernetes node if applicable
+      ansible.builtin.command: >
+        kubectl drain {{ inventory_hostname }}
+        --ignore-daemonsets
+        --delete-emptydir-data
+        --force
+        --timeout=180s
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      failed_when: false
+      changed_when: true
+
     - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 3600
+      environment:
+        DEBIAN_FRONTEND: noninteractive
       when: ansible_os_family == "Debian"
 
     - name: Perform dist-upgrade
@@ -16,6 +52,9 @@
         upgrade: dist
         autoremove: true
         autoclean: true
+        dpkg_options: 'force-confdef,force-confold'
+      environment:
+        DEBIAN_FRONTEND: noninteractive
       when: ansible_os_family == "Debian"
 
     - name: Check if system reboot is required
@@ -28,16 +67,18 @@
       block:
         - name: Reboot host gracefully
           ansible.builtin.reboot:
-            msg: "Automated rolling maintenance reboot via AWX"
+            msg: "Automated rolling maintenance reboot via Snarfnet Automation"
             reboot_timeout: 300
             connect_timeout: 30
             test_command: uptime
 
-        - name: Verify systemd-timesyncd is active after reboot
-          ansible.builtin.systemd:
-            name: systemd-timesyncd
-            state: started
-          ignore_errors: true
+        - name: Verify systemd-timesyncd or chrony is active after reboot
+          ansible.builtin.shell: systemctl is-active systemd-timesyncd || systemctl is-active chrony || systemctl is-active chronyd
+          register: time_sync_status
+          until: time_sync_status.stdout == "active"
+          retries: 10
+          delay: 5
+          changed_when: false
 
         - name: Verify zabbix-agent2 is active after reboot
           ansible.builtin.systemd:
@@ -45,3 +86,25 @@
             state: started
           ignore_errors: true
 
+    - name: Uncordon Kubernetes node if applicable
+      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      changed_when: true
+
+    - name: Wait for Kubernetes node to report Ready
+      ansible.builtin.command: >
+        kubectl get node {{ inventory_hostname }}
+        -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      register: node_ready
+      until: node_ready.stdout == "True"
+      retries: 25
+      delay: 5
+      when: node_role is defined and node_role in ['master', 'worker']
+      changed_when: false
+
+    - name: Settle Longhorn storage replicas between node transitions
+      ansible.builtin.pause:
+        seconds: 15
+      when: node_role is defined and node_role in ['master', 'worker']

docs/infrastructure_improvement_plan.md

@@ -41,7 +41,7 @@ Our automated live probe across the homelab APIs identified the following baseli
 | **Aerodrome K3s (`local`)** | All 10 Nodes (`ctl01-03` & `wrk01-07`): **`v1.30.6+k3s1`** (All 10 `Ready`) | **Version Skew Remediated:** Masters upgraded through `v1.27.16` $\rightarrow$ `v1.28.15` $\rightarrow$ `v1.29.10` $\rightarrow$ `v1.30.6+k3s1` with zero workload downtime. Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active. Enrolled in ArgoCD. |
 | **Spaceport K3s (`c-m-4fgkl75b`)** | 4 Masters + 6 Workers: **`v1.31.4+k3s1`** (All 10 `Ready`) | Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active across Gitea, Zabbix Postgres, Jenkins, and Semaphore DBs. ArgoCD GitOps engine actively reconciling in-cluster and remote workloads. |
 | **Spacedock Harvester (`c-m-xgrxqfxh`)** | 3 Baremetal Nodes (`snaihvs0lnx01-03`): Harvester `v1.8.0` / `v1.35.2+rke2r1` | Healthy `harvester-longhorn` default StorageClass. Declarative VLAN networks and VM templates codified in Terraform. Monitored via SNMP / ICMP. |
-| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | NPM pinned in `PublishMode: host` on `.74` with daily NAS backup. Hardened NFS persistent mounts (`/mnt/nfs_shares`) on Synology NAS (`14.10.10.71:/volume1/docker`). Automated SQLite backup timer active with daily tarball sync to NAS. |
+| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | Dual-NPM HA Keepalived cluster active on VIP `140.44.4.70` across `snshpd4lnxswm01` (.71) and `snshpd4lnxswm02` (.72) in `PublishMode: host`. Hardened NFS persistent mounts (`/mnt/nfs_shares`) on Synology NAS (`14.10.10.71:/volume1/docker`). Automated SQLite backup timer active with daily tarball sync to NAS. |
 | **Zabbix (`zabbix.snarfnet.net`)** | Version `7.0.16` (`140.44.4.57:8888`, Trapper `140.44.4.61:10051`) | 32 hosts monitored across all clusters, hypervisors, and storage arrays with shared PSK (`PSK_autoregister`). Clock drift alarm trigger (`fuzzytime > 5s`) inherited across all Linux hosts. Gotify & Discord problem actions active. |
 | **Gotify (`gotify.snarfnet.net`)** | Version `2.9.1` (`140.44.4.70:8090`) | 5 structured notification channels provisioned. Internal VIP (`http://140.44.4.70:8090`) active for direct programmatic alerts. `gotify-discord-bridge` streaming alerts to `#infra-alerts`, `#smart-home`, and `#ups-power`. |
 | **AWX (`awx.snarfnet.net`)** | Operator `2.19.1` / AWX `24.6.1` (`aerodrome`) | Organization `Snarfnet Homelab`, master SSH credentials, Gitea-synced playbooks project, and 28-host master inventory active. Daily drift audit and rolling upgrade job templates verified green. |
@@ -399,7 +399,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
 | Role | Hostname / Identifier | Target IP | Subnet / VLAN | Description & Failure Protocol |
 |---|---|---|---|---|
 | **NUT Master Server** | `nutserver.snarfnet.net` | `14.4.4.44` | VLAN 50 (Smart Home) | Primary NUT `upsd` daemon polling physical UPS via USB/Serial/SNMP. Runs `upsmon master` and holds `/root/.ssh/id_ed25519_snarfnet`. |
-| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.74` | VLAN 20 (Swarm 04) | NPM running in `PublishMode: host` on ports 80/443/81. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
+| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.70` | VLAN 20 (Keepalived VIP) | Dual-NPM active/standby running in `PublishMode: host` on ports 80/443/81 across Swarm 01 & 02. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
 | **Authentication & SSO** | `authentik.snarfnet.net` | `140.44.4.29:444` | VLAN 20 (Aerodrome K3s) | Handles Forward Auth & Conditional MFA. Runs on Longhorn storage in Aerodrome. |
 | **UPS Web Dashboard** | `peanut.snarfnet.net` | `140.44.4.70:8050` | VLAN 20 (Swarm VIP) | Peanut container UI consuming `14.4.4.44:3493`. Protected by Authentik Forward Auth externally. |
 | **Emergency Push & Discord** | `gotify.snarfnet.net` | `140.44.4.70:8090` | VLAN 20 (Swarm VIP) | Gotify server + `gotify-discord-bridge`. **Internal outage alerts use `http://140.44.4.70:8090`** (+ direct Discord webhook fallback) to bypass Authentik during shutdown. |
@@ -466,7 +466,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
 3. **Stage 3 (NUT Master):** `14.4.4.44` boots (`BIOS: Restore on AC Power Loss = Always On`), starts `upsd`, and sends `ONLINE` alert to Discord/Gotify.
 4. **Stage 4 (Storage Gate — Must Complete Before Compute):** Synology NAS arrays (`14.10.10.71` & `14.10.10.72`) boot, mount Btrfs pools, and expose NFS (`:2049`).
 5. **Stage 5 (Control Planes & Hypervisors):** Harvester (`140.44.4.81-83`), K3s Masters (`aerodrome` & `spaceport`), and Swarm Managers (`140.44.4.71-75`) boot and establish etcd/Raft quorum.
-6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and NPM on `snshpd4lnxswm04` (`140.44.4.74`) binds host ports `80/443/81`.
+6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and Dual-NPM on Swarm 01 & 02 (`140.44.4.71-72`) binds host ports `80/443/81` behind Keepalived VIP `140.44.4.70`.
 
 ---
 

scripts/check_db_creds.py

@@ -0,0 +1,30 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("--- Testing authentik dump ---")
+cmd = "kubectl exec -n infra authentik-postgresql-0 -- env"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("--- Testing gitea env ---")
+cmd = "kubectl exec -n infrastructure gitea-postgresql-0 -- env"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+
+print("--- Testing zabbix env ---")
+cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- env"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_host_ports.py

@@ -0,0 +1,21 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    for ip, node in [('140.44.4.71', 'snshpd4lnxswm01'), ('140.44.4.72', 'snshpd4lnxswm02')]:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        
+        print(f"=== {node} ({ip}) ===")
+        stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:81 |:443 " || true')
+        print("SS:\n", stdout.read().decode())
+        stdin, stdout, stderr = ssh.exec_command('sudo docker ps -a --filter "name=nginx" --format "{{.ID}} {{.Status}} {{.Names}}"')
+        print("Containers:\n", stdout.read().decode())
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/check_keepalived_all.py

@@ -0,0 +1,35 @@
+import paramiko
+
+NODES = [
+    ("swm01", "140.44.4.71"),
+    ("swm02", "140.44.4.72"),
+    ("swm03", "140.44.4.73"),
+    ("swm04", "140.44.4.74"),
+    ("swm05", "140.44.4.75"),
+]
+
+def main():
+    for name, ip in NODES:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
+            stdin, stdout, stderr = ssh.exec_command("systemctl is-active keepalived 2>/dev/null || echo inactive")
+            status = stdout.read().decode().strip()
+            
+            stdin, stdout, stderr = ssh.exec_command("ip -4 addr show | grep '140.44.4.70' || true")
+            vip = stdout.read().decode().strip()
+            
+            stdin, stdout, stderr = ssh.exec_command("grep -E 'state|priority' /etc/keepalived/keepalived.conf 2>/dev/null || true")
+            conf = stdout.read().decode().strip()
+            
+            print(f"[{name} - {ip}] Keepalived: {status} | VIP: {'YES' if '140.44.4.70' in vip else 'NO'}")
+            if conf:
+                print(f"   Config: {conf.splitlines()}")
+            ssh.close()
+        except Exception as e:
+            print(f"[{name} - {ip}] Error:", e)
+
+if __name__ == '__main__':
+    main()
+

scripts/check_longhorn_settings.py

@@ -0,0 +1,29 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    for name, ip in [("aerodrome", "140.44.4.11"), ("spaceport", "140.44.4.41")]:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        
+        print(f"=== LONGHORN SETTINGS ON {name} ({ip}) ===")
+        settings = [
+            'auto-salvage-volume',
+            'detach-manually-attached-volume-with-missing-node',
+            'replica-replenishment-wait-interval',
+            'pod-deletion-for-node-down-failure-handling',
+            'backup-target'
+        ]
+        for s in settings:
+            cmd = f"sudo kubectl get setting.longhorn.io {s} -n longhorn-system -o jsonpath='{{.value}}' 2>/dev/null || echo 'not found'"
+            stdin, stdout, stderr = ssh.exec_command(cmd)
+            val = stdout.read().decode().strip()
+            print(f"  {s}: {val}")
+            
+        ssh.close()
+
+if __name__ == '__main__':
+    main()

scripts/check_nfs_mounts.py

@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+    cmd = "df -h | grep -i nfs || echo 'no nfs mounts'"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"[{name.upper()}] NFS mounts:\n{stdout.read().decode().strip()}")
+    ssh.close()
+

scripts/check_npm_status.py

@@ -0,0 +1,27 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    for ip in ['140.44.4.71', '140.44.4.72']:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        
+        stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:443 |:81 " || true')
+        print(f"=== LISTENING PORTS ON {ip} ===")
+        print(stdout.read().decode())
+        
+        stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" --format "{{.ID}} {{.Status}}"')
+        cid = stdout.read().decode().strip()
+        print(f"Container on {ip}: {cid}")
+        if cid:
+            cid_clean = cid.split()[0]
+            stdin, stdout, stderr = ssh.exec_command(f'sudo docker logs {cid_clean} --tail 15')
+            print(f"Logs on {ip}:\n{stdout.read().decode()}")
+            
+        ssh.close()
+
+if __name__ == '__main__':
+    main()

scripts/check_npm_volumes.py

@@ -0,0 +1,16 @@
+import paramiko
+
+def main():
+    for ip in ['140.44.4.71', '140.44.4.72']:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        stdin, stdout, stderr = ssh.exec_command("sudo docker volume inspect nginx-proxy-manager_nginx-data --format '{{json .Options}}'")
+        print(f"Volume data on {ip}: {stdout.read().decode().strip()}")
+        stdin, stdout, stderr = ssh.exec_command("sudo docker volume inspect nginx-proxy-manager_nginx-letsencrypt --format '{{json .Options}}'")
+        print(f"Volume letsencrypt on {ip}: {stdout.read().decode().strip()}")
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/check_port_forwards.py

@@ -0,0 +1,16 @@
+import paramiko
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        ssh.connect('14.44.4.1', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
+        stdin, stdout, stderr = ssh.exec_command('iptables -t nat -S | grep -E "80|443"')
+        print("IPTABLES NAT:\n", stdout.read().decode())
+        ssh.close()
+    except Exception as e:
+        print("SSH to UDM Pro error:", e)
+
+if __name__ == '__main__':
+    main()
+

scripts/check_swarm_ports.py

@@ -0,0 +1,15 @@
+import paramiko
+
+def main():
+    for ip in ['140.44.4.71', '140.44.4.72']:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:443 |:81 " || true')
+        print(f"=== PORTS ON {ip} ===")
+        print(stdout.read().decode())
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/check_swarm_service_status.py

@@ -0,0 +1,23 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== DOCKER SERVICE LS ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker service ls')
+    print(stdout.read().decode())
+
+    print("=== DOCKER SERVICE PS ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app')
+    print(stdout.read().decode())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/deploy_dual_npm.py

@@ -0,0 +1,60 @@
+import paramiko
+import time
+import urllib.request
+import ssl
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== STEP 1: Updating nginx-proxy-manager_app to Dual-Node Swarm Placement ===")
+    cmd = (
+        "sudo docker service update "
+        "--constraint-rm 'node.hostname == snshpd4lnxswm04' "
+        "--constraint-add 'node.labels.npm == true' "
+        "--replicas 2 "
+        "--update-order start-first "
+        "nginx-proxy-manager_app"
+    )
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print("Service update output:\n", stdout.read().decode())
+    print("Service update stderr:\n", stderr.read().decode())
+
+    print("=== STEP 2: Waiting for tasks to stabilize ===")
+    time.sleep(10)
+    
+    stdin, stdout, stderr = ssh.exec_command("sudo docker service ps nginx-proxy-manager_app --no-trunc")
+    print("Tasks:\n", stdout.read().decode())
+
+    ssh.close()
+
+    print("=== STEP 3: Verifying HTTP / HTTPS endpoints ===")
+    for ip in ['140.44.4.70', '140.44.4.71', '140.44.4.72']:
+        # Port 80
+        try:
+            req = urllib.request.Request(f"http://{ip}", headers={"Host": "test.snarfnet.net"})
+            with urllib.request.urlopen(req, timeout=4) as r:
+                print(f"[{ip}:80] HTTP {r.status}")
+        except urllib.error.HTTPError as e:
+            print(f"[{ip}:80] HTTP {e.code} (NPM responding)")
+        except Exception as e:
+            print(f"[{ip}:80] Error: {e}")
+
+        # Port 81 (Admin UI)
+        try:
+            req = urllib.request.Request(f"http://{ip}:81")
+            with urllib.request.urlopen(req, timeout=4) as r:
+                print(f"[{ip}:81] HTTP {r.status} (Admin UI responding)")
+        except urllib.error.HTTPError as e:
+            print(f"[{ip}:81] HTTP {e.code}")
+        except Exception as e:
+            print(f"[{ip}:81] Error: {e}")
+
+if __name__ == '__main__':
+    main()
+

scripts/dump_k8s_databases.py

@@ -0,0 +1,65 @@
+import paramiko
+import sys
+import datetime
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+db_targets = [
+    {
+        "cluster": "aerodrome",
+        "ip": "140.44.4.11",
+        "namespace": "infra",
+        "pod": "authentik-postgresql-0",
+        "name": "authentik",
+        "dump_cmd": "kubectl exec -n infra authentik-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/authentik_db_{timestamp}.sql.gz"
+    },
+    {
+        "cluster": "aerodrome",
+        "ip": "140.44.4.11",
+        "namespace": "awx",
+        "pod": "awx-postgres-15-0",
+        "name": "awx",
+        "dump_cmd": "kubectl exec -n awx awx-postgres-15-0 -- pg_dumpall -U postgres | gzip > /var/backups/awx_db_{timestamp}.sql.gz"
+    },
+    {
+        "cluster": "spaceport",
+        "ip": "140.44.4.41",
+        "namespace": "infrastructure",
+        "pod": "gitea-postgresql-0",
+        "name": "gitea",
+        "dump_cmd": "kubectl exec -n infrastructure gitea-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/gitea_db_{timestamp}.sql.gz"
+    },
+    {
+        "cluster": "spaceport",
+        "ip": "140.44.4.41",
+        "namespace": "monitoring",
+        "pod": "zabbix-postgresql-0",
+        "name": "zabbix",
+        "dump_cmd": "kubectl exec -n monitoring zabbix-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/zabbix_db_{timestamp}.sql.gz"
+    }
+]
+
+for target in db_targets:
+    cluster = target["cluster"]
+    ip = target["ip"]
+    name = target["name"]
+    print(f"[{cluster.upper()}] Backing up {name} database...")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=15)
+    
+    cmd = f"mkdir -p /var/backups && {target['dump_cmd'].replace('{timestamp}', timestamp)}"
+    stdin, stdout, stderr = ssh.exec_command(cmd, timeout=300)
+    exit_status = stdout.channel.recv_exit_status()
+    err = stderr.read().decode().strip()
+    
+    if exit_status == 0:
+        check_cmd = f"ls -lh /var/backups/{name}_db_{timestamp}.sql.gz"
+        _, chk_out, _ = ssh.exec_command(check_cmd)
+        print(f"  SUCCESS: {chk_out.read().decode().strip()}")
+    else:
+        print(f"  FAILED (exit {exit_status}): {err}")
+        
+    ssh.close()

scripts/inspect_container_nginx.py

@@ -0,0 +1,25 @@
+import paramiko
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" -q')
+    cid = stdout.read().decode().strip()
+    print("Container ID:", cid)
+    
+    if cid:
+        stdin, stdout, stderr = ssh.exec_command(f'sudo docker exec {cid} nginx -t')
+        print("NGINX -t STDOUT:\n", stdout.read().decode())
+        print("NGINX -t STDERR:\n", stderr.read().decode())
+        
+        stdin, stdout, stderr = ssh.exec_command(f'sudo docker exec {cid} curl -I http://127.0.0.1:80')
+        print("CURL 80 INSIDE:\n", stdout.read().decode())
+        print("CURL 80 STDERR:\n", stderr.read().decode())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/inspect_db_env.py

@@ -0,0 +1,29 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== AERODROME DB USERS/ENV ===")
+for ns, pod in [('infra', 'authentik-postgresql-0'), ('awx', 'awx-postgres-15-0')]:
+    cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
+
+ssh.close()
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+
+print("=== SPACEPORT DB USERS/ENV ===")
+for ns, pod in [('infrastructure', 'gitea-postgresql-0'), ('monitoring', 'zabbix-postgresql-0')]:
+    cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
+
+ssh.close()
+

scripts/inspect_db_pods.py

@@ -0,0 +1,19 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
+    cmd = "kubectl get pods -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase"
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    lines = stdout.read().decode().splitlines()
+    print(f"=== {name.upper()} PODS ===")
+    for line in lines:
+        lower = line.lower()
+        if any(k in lower for k in ['postgres', 'mysql', 'authentik', 'gitea', 'zabbix', 'awx', 'semaphore']):
+            print("  " + line)
+    ssh.close()
+

scripts/inspect_keepalived_conf.py

@@ -0,0 +1,26 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== /etc/keepalived/keepalived.conf ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo cat /etc/keepalived/keepalived.conf')
+    print(stdout.read().decode())
+
+    print("=== /etc/keepalived/scripts/ ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo ls -la /etc/keepalived/scripts/ || true')
+    print(stdout.read().decode())
+
+    stdin, stdout, stderr = ssh.exec_command('sudo cat /etc/keepalived/scripts/* || true')
+    print(stdout.read().decode())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/inspect_npm_72.py

@@ -0,0 +1,34 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== DOCKER EXEC PS INSIDE f026d1c24ef6 ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker exec f026d1c24ef6 ps aux')
+    print(stdout.read().decode())
+    print(stderr.read().decode())
+
+    print("=== REDIRECTION_HOST LS ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo ls -la /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/redirection_host || true')
+    print(stdout.read().decode())
+    print(stderr.read().decode())
+
+    print("=== RECENT DOCKER LOGS WITH TIMESTAMPS ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker logs f026d1c24ef6 --tail 30')
+    print(stdout.read().decode())
+
+    print("=== LOCAL CURL TEST ON 140.44.4.72 ===")
+    stdin, stdout, stderr = ssh.exec_command('curl -I http://127.0.0.1:80 ; curl -I http://127.0.0.1:81')
+    print(stdout.read().decode())
+    print(stderr.read().decode())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/inspect_swarm_nodes_labels.py

@@ -0,0 +1,25 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    stdin, stdout, stderr = ssh.exec_command('sudo docker node ls --format "{{.Hostname}}"')
+    nodes = stdout.read().decode().split()
+    for n in nodes:
+        stdin, stdout, stderr = ssh.exec_command(f'sudo docker node inspect {n} --format "{{{{.Description.Hostname}}}}: {{{{.Spec.Labels}}}}"')
+        print(stdout.read().decode().strip())
+
+    print("\n=== DOCKER SERVICE PS ALL ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app --no-trunc')
+    print(stdout.read().decode())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/list_longhorn_settings.py

@@ -0,0 +1,21 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    stdin, stdout, stderr = ssh.exec_command('kubectl get settings.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,VALUE:.value')
+    lines = stdout.read().decode().splitlines()
+    for l in lines:
+        for kw in ['salvage', 'detach', 'replenish', 'pod', 'node-down']:
+            if kw in l.lower():
+                print(l)
+                break
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/pin_spaceport_vips.py

@@ -51,3 +51,4 @@ def main():
 
 if __name__ == '__main__':
     main()
+

scripts/rescale_npm.py

@@ -0,0 +1,30 @@
+import sys
+import time
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== SCALING TO 1 ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker service scale nginx-proxy-manager_app=1')
+    print(stdout.read().decode())
+    time.sleep(3)
+
+    print("=== SCALING TO 2 ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker service scale nginx-proxy-manager_app=2')
+    print(stdout.read().decode())
+    time.sleep(5)
+
+    print("=== SERVICE PS ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app --filter desired-state=running')
+    print(stdout.read().decode())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/setup_dual_npm_cluster.py

@@ -0,0 +1,24 @@
+import paramiko
+import json
+import time
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== STEP 1: Add node labels 'npm=true' to swm01 (.71) and swm02 (.72) ===")
+    stdin, stdout, stderr = ssh.exec_command('sudo docker node update --label-add npm=true snshpd4lnxswm01 && sudo docker node update --label-add npm=true snshpd4lnxswm02')
+    print("Labels update:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+    # Check node labels
+    stdin, stdout, stderr = ssh.exec_command("sudo docker node inspect snshpd4lnxswm01 --format '{{json .Spec.Labels}}'")
+    print("Labels swm01:", stdout.read().decode().strip())
+    stdin, stdout, stderr = ssh.exec_command("sudo docker node inspect snshpd4lnxswm02 --format '{{json .Spec.Labels}}'")
+    print("Labels swm02:", stdout.read().decode().strip())
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/sync_backups_to_nas.py

@@ -0,0 +1,37 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+nodes = [
+    ("aerodrome", "140.44.4.11", ["authentik_clean.sql.gz", "awx_clean.sql.gz"]),
+    ("spaceport", "140.44.4.41", ["gitea_clean.sql.gz", "zabbix_clean.sql.gz"])
+]
+
+for name, ip, files in nodes:
+    print(f"[{name.upper()}] Mounting NAS and copying backups...")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    setup_cmd = """
+    mkdir -p /mnt/nas_backup
+    mount -t nfs -o vers=4 14.10.10.71:/volume1/docker /mnt/nas_backup || mount -t nfs 14.10.10.71:/volume1/docker /mnt/nas_backup
+    mkdir -p /mnt/nas_backup/backups/databases
+    """
+    stdin, stdout, stderr = ssh.exec_command(setup_cmd)
+    stdout.channel.recv_exit_status()
+    
+    for f in files:
+        cp_cmd = f"cp /var/backups/{f} /mnt/nas_backup/backups/databases/ && ls -lh /mnt/nas_backup/backups/databases/{f}"
+        stdin, stdout, stderr = ssh.exec_command(cp_cmd)
+        stdout.channel.recv_exit_status()
+        print(f"  Saved to NAS: {stdout.read().decode().strip()}")
+        
+    umount_cmd = "umount -f /mnt/nas_backup || true"
+    stdin, stdout, stderr = ssh.exec_command(umount_cmd)
+    stdout.channel.recv_exit_status()
+    ssh.close()
+
+print("All database backups verified mirrored to Synology NAS (14.10.10.71).")
+

scripts/test_authentik_dump.py

@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U authentik authentik' | gzip > /var/backups/authentik_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print("STDERR:", stderr.read().decode())
+_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
+print("RESULT:", out.read().decode().strip())
+ssh.close()
+

scripts/test_clean_db_dumps.py

@@ -0,0 +1,40 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+# Authentik
+print("=== Backing up Authentik ===")
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/postgres-password) pg_dump -U postgres authentik' | gzip > /var/backups/authentik_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+err = stderr.read().decode().strip()
+if err: print("Authentik err:", err)
+_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
+print("Authentik size:", out.read().decode().strip())
+ssh.close()
+
+# Spaceport (Gitea & Zabbix)
+print("=== Backing up Gitea & Zabbix ===")
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = "mkdir -p /var/backups && kubectl exec -n infrastructure gitea-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U ugandanese baptiste' | gzip > /var/backups/gitea_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+err = stderr.read().decode().strip()
+if err: print("Gitea err:", err)
+_, out, _ = ssh.exec_command("ls -lh /var/backups/gitea_clean.sql.gz")
+print("Gitea size:", out.read().decode().strip())
+
+cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- sh -c 'PGPASSWORD=zabbix pg_dump -U zabbix zabbix' | gzip > /var/backups/zabbix_clean.sql.gz"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+err = stderr.read().decode().strip()
+if err: print("Zabbix err:", err)
+_, out, _ = ssh.exec_command("ls -lh /var/backups/zabbix_clean.sql.gz")
+print("Zabbix size:", out.read().decode().strip())
+
+ssh.close()
+

scripts/test_etcd_snapshots.py

@@ -0,0 +1,35 @@
+import paramiko
+import sys
+import datetime
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+clusters = [
+    ("aerodrome", "140.44.4.11"),
+    ("spaceport", "140.44.4.41")
+]
+
+for name, ip in clusters:
+    print(f"[{name.upper()}] Connecting to {ip}...")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    cmd = f"k3s etcd-snapshot save --name pre-upgrade-{name}-{timestamp}"
+    print(f"[{name.upper()}] Executing: {cmd}")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    out = stdout.read().decode().strip()
+    err = stderr.read().decode().strip()
+    if out:
+        print(f"[{name.upper()}] STDOUT: {out}")
+    if err:
+        print(f"[{name.upper()}] STDERR: {err}")
+        
+    check_cmd = "k3s etcd-snapshot ls"
+    stdin, stdout, stderr = ssh.exec_command(check_cmd)
+    print(f"[{name.upper()}] Recent Snapshots:\n" + stdout.read().decode().strip())
+    
+    ssh.close()
+    print("-" * 50)

scripts/test_keepalived_failover.py

@@ -0,0 +1,67 @@
+import sys
+import time
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def check_vip(ssh):
+    stdin, stdout, stderr = ssh.exec_command("ip -4 addr show dev eth0 | grep '140.44.4.70' || true")
+    return bool(stdout.read().decode().strip())
+
+def curl_vip(ssh):
+    stdin, stdout, stderr = ssh.exec_command("curl -s -o /dev/null -w '%{http_code}' -m 2 http://140.44.4.70:80 || true")
+    return stdout.read().decode().strip()
+
+def main():
+    ssh1 = paramiko.SSHClient()
+    ssh1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh1.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    ssh2 = paramiko.SSHClient()
+    ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh2.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== INITIAL STATE ===")
+    v1 = check_vip(ssh1)
+    v2 = check_vip(ssh2)
+    code = curl_vip(ssh1)
+    print(f"swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
+
+    print("\n=== TRIGGERING FAILOVER: STOPPING KEEPALIVED ON swm01 ===")
+    ssh1.exec_command("sudo systemctl stop keepalived")
+    
+    # Wait up to 5 seconds for swm02 to claim VIP
+    claimed = False
+    for i in range(5):
+        time.sleep(1)
+        if check_vip(ssh2):
+            claimed = True
+            break
+            
+    v1 = check_vip(ssh1)
+    v2 = check_vip(ssh2)
+    code = curl_vip(ssh2)
+    print(f"Failover result -> swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
+
+    print("\n=== RESTORING swm01: STARTING KEEPALIVED ON swm01 ===")
+    ssh1.exec_command("sudo systemctl start keepalived")
+    
+    # Wait up to 5 seconds for swm01 to reclaim VIP
+    reclaimed = False
+    for i in range(5):
+        time.sleep(1)
+        if check_vip(ssh1):
+            reclaimed = True
+            break
+            
+    v1 = check_vip(ssh1)
+    v2 = check_vip(ssh2)
+    code = curl_vip(ssh1)
+    print(f"Fallback result -> swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
+
+    ssh1.close()
+    ssh2.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/test_npm_both_nodes.py

@@ -0,0 +1,26 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    for ip, node in [('140.44.4.71', 'snshpd4lnxswm01'), ('140.44.4.72', 'snshpd4lnxswm02')]:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        
+        print(f"=== TESTING {node} ({ip}) ===")
+        stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" --format "{{.ID}} {{.Status}} {{.Names}}"')
+        print("Containers:", stdout.read().decode().strip())
+        
+        stdin, stdout, stderr = ssh.exec_command('curl -I http://127.0.0.1:80 ; curl -I http://127.0.0.1:81')
+        print("Curls:\n", stdout.read().decode())
+        err = stderr.read().decode()
+        if err:
+            print("Errors:\n", err)
+            
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/test_worker_os_patch.py

@@ -0,0 +1,113 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+MASTER_IP = '140.44.4.11'
+WORKER_IP = '140.44.4.27'
+NODE_NAME = 'sninfr2lnxwrk07'
+KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+print(f"=== STEP 1: TEST ROLLING OS PATCH ON {NODE_NAME} ({WORKER_IP}) ===")
+
+# 1. Connect to master to check pods on node
+master_ssh = paramiko.SSHClient()
+master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
+
+print(f"[*] Checking active pods on {NODE_NAME}...")
+_, stdout, _ = master_ssh.exec_command(f"kubectl get pods -A --field-selector spec.nodeName={NODE_NAME}")
+print(stdout.read().decode())
+
+print(f"[*] Cordoning {NODE_NAME}...")
+_, stdout, stderr = master_ssh.exec_command(f"kubectl cordon {NODE_NAME}")
+print("  " + stdout.read().decode().strip())
+
+print(f"[*] Draining {NODE_NAME}...")
+_, stdout, stderr = master_ssh.exec_command(f"kubectl drain {NODE_NAME} --ignore-daemonsets --delete-emptydir-data --force --timeout=180s")
+print("  " + stdout.read().decode().strip())
+err = stderr.read().decode().strip()
+if err:
+    print("  (drain notice):", err)
+
+# 2. Connect to worker to perform apt-get update & dist-upgrade
+print(f"[*] Connecting directly to worker {WORKER_IP} for package upgrades...")
+worker_ssh = paramiko.SSHClient()
+worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=10)
+
+upgrade_cmd = "DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade -y && apt-get autoremove -y"
+print("[*] Running dist-upgrade (preserving configurations)...")
+stdin, stdout, stderr = worker_ssh.exec_command(upgrade_cmd)
+exit_status = stdout.channel.recv_exit_status()
+print(f"[*] dist-upgrade completed with exit code: {exit_status}")
+
+# 3. Check reboot requirement
+_, stdout, _ = worker_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
+reboot_needed = stdout.read().decode().strip() == 'yes'
+print(f"[*] Reboot required: {reboot_needed}")
+
+if reboot_needed:
+    print("[*] Rebooting worker node...")
+    try:
+        worker_ssh.exec_command("reboot")
+    except Exception:
+        pass
+    worker_ssh.close()
+    
+    print("[*] Waiting 30 seconds for host to reboot...")
+    time.sleep(30)
+    
+    # Poll SSH until online
+    reconnected = False
+    for attempt in range(25):
+        try:
+            worker_ssh = paramiko.SSHClient()
+            worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=6)
+            reconnected = True
+            print(f"[*] Worker reconnected successfully on attempt {attempt+1}!")
+            break
+        except Exception:
+            time.sleep(5)
+            
+    if not reconnected:
+        print("[!] ERROR: Worker node did not come back online within timeout!")
+        sys.exit(1)
+        
+    # Verify services
+    _, stdout, _ = worker_ssh.exec_command("systemctl is-active systemd-timesyncd || systemctl is-active chrony")
+    print(f"  TimeSync: {stdout.read().decode().strip()}")
+    _, stdout, _ = worker_ssh.exec_command("systemctl is-active zabbix-agent2")
+    print(f"  ZabbixAgent2: {stdout.read().decode().strip()}")
+    _, stdout, _ = worker_ssh.exec_command("systemctl is-active k3s-agent || systemctl is-active k3s")
+    print(f"  K3sAgent: {stdout.read().decode().strip()}")
+    worker_ssh.close()
+else:
+    worker_ssh.close()
+
+# 4. Uncordon node
+print(f"[*] Uncordoning {NODE_NAME}...")
+_, stdout, _ = master_ssh.exec_command(f"kubectl uncordon {NODE_NAME}")
+print("  " + stdout.read().decode().strip())
+
+# 5. Wait for node to be Ready
+print(f"[*] Verifying node status in Kubernetes...")
+for _ in range(15):
+    _, stdout, _ = master_ssh.exec_command(f"kubectl get node {NODE_NAME} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'")
+    status = stdout.read().decode().strip()
+    if status == "True":
+        print(f"[OK] Node {NODE_NAME} is fully Ready!")
+        break
+    time.sleep(4)
+
+# 6. Check Longhorn volume robustness
+print("[*] Settling Longhorn volume replicas...")
+time.sleep(10)
+_, stdout, _ = master_ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness")
+print(stdout.read().decode())
+
+master_ssh.close()
+print(f"[SUCCESS] Step 1 Test Worker OS Patch on {NODE_NAME} completed flawlessly!")
+

scripts/tune_longhorn_settings.py

@@ -0,0 +1,34 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    clusters = [
+        ("aerodrome", "140.44.4.11"),
+        ("spaceport", "140.44.4.41")
+    ]
+    for name, host in clusters:
+        print(f"=== Configuring Longhorn settings on {name} ({host}) ===")
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(host, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+        # 1. replica-replenishment-wait-interval -> 300
+        cmd1 = "kubectl patch setting.longhorn.io replica-replenishment-wait-interval -n longhorn-system --type=merge -p '{\"value\":\"300\"}'"
+        stdin, stdout, stderr = ssh.exec_command(cmd1)
+        print("replica-replenishment-wait-interval:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+        # 2. node-down-pod-deletion-policy -> delete-deployment-pod
+        cmd2 = "kubectl patch setting.longhorn.io node-down-pod-deletion-policy -n longhorn-system --type=merge -p '{\"value\":\"delete-deployment-pod\"}'"
+        stdin, stdout, stderr = ssh.exec_command(cmd2)
+        print("node-down-pod-deletion-policy:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+        # Verify
+        stdin, stdout, stderr = ssh.exec_command("kubectl get setting.longhorn.io replica-replenishment-wait-interval node-down-pod-deletion-policy -n longhorn-system -o custom-columns=NAME:.metadata.name,VALUE:.value")
+        print("VERIFIED:\n", stdout.read().decode().strip())
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
+

scripts/update_keepalived_healthcheck.py

@@ -0,0 +1,50 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+NODES = [
+    ("swm01", "140.44.4.71"),
+    ("swm02", "140.44.4.72"),
+    ("swm03", "140.44.4.73"),
+    ("swm04", "140.44.4.74"),
+    ("swm05", "140.44.4.75"),
+]
+
+SCRIPT_CONTENT = """#!/bin/bash
+# Check docker daemon health
+systemctl is-active --quiet docker || exit 1
+
+# If NPM container is running on this node, verify HTTP responsiveness
+if docker ps --format '{{.Names}}' 2>/dev/null | grep -q "nginx-proxy-manager_app"; then
+    curl -sf -m 2 http://127.0.0.1:80 >/dev/null 2>&1 || curl -sf -m 2 http://127.0.0.1:81 >/dev/null 2>&1 || exit 1
+fi
+
+exit 0
+"""
+
+def main():
+    for name, ip in NODES:
+        print(f"=== Updating {name} ({ip}) ===")
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        
+        # Write to temp file then sudo mv
+        sftp = ssh.open_sftp()
+        with sftp.open('/tmp/check_docker.sh', 'w') as f:
+            f.write(SCRIPT_CONTENT.replace('\r\n', '\n'))
+        sftp.close()
+        
+        cmd = "sudo mv /tmp/check_docker.sh /etc/keepalived/scripts/check_docker.sh && sudo chmod +x /etc/keepalived/scripts/check_docker.sh && sudo /etc/keepalived/scripts/check_docker.sh && echo 'TEST: OK'"
+        stdin, stdout, stderr = ssh.exec_command(cmd)
+        print(stdout.read().decode().strip())
+        err = stderr.read().decode().strip()
+        if err:
+            print("ERR:", err)
+            
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
+