feat: add infrastructure automation and diagnostic scripts for Zabbix, Nginx, and Authentik

docs(topology): regenerate high-res PNG diagram to reflect latest layout, hardware and storage metrics
chore: update homelab-gitops-repo submodule reference
2026-11-12 20:18:37 -05:00 · 2026-11-12 17:13:51 -05:00 · 2026-11-12 17:09:51 -05:00 · 2026-11-12 17:08:14 -05:00 · 2026-11-12 16:57:08 -05:00 · 2026-11-08 15:03:27 -05:00
978 changed files with 30995 additions and 107 deletions
--- a/ansible/playbooks/upgrade_k3s_aerodrome.yml
+++ b/ansible/playbooks/upgrade_k3s_aerodrome.yml
@@ -1,10 +1,10 @@
 ---
- name: Aerodrome K3s Control Plane Sequential Minor Version Upgrade
+- name: Aerodrome K3s Control Plane Sequential Upgrade
  hosts: k3s_aerodrome_masters
  gather_facts: false
  serial: 1
  vars:
-    target_k3s_version: "v1.27.16+k3s1"
+    target_k3s_version: "v1.31.4+k3s1"

  tasks:
    - name: Query pre-upgrade K3s version
@@ -16,7 +16,7 @@
      ansible.builtin.debug:
        msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"

-    - name: Download target K3s release binary and replace executable
+    - name: Download target K3s release binary
      ansible.builtin.shell: |
        set -e
        ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
@@ -36,12 +36,12 @@
      ansible.builtin.command: systemctl is-active k3s
      register: k3s_active
      until: k3s_active.stdout == "active"
-      retries: 20
+      retries: 25
      delay: 4
      changed_when: false
      when: target_k3s_version not in pre_k3s_ver.stdout

-    - name: Wait for upgraded node to report Ready in Kubernetes
+    - name: Wait for upgraded master to report Ready in Kubernetes
      ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
      delegate_to: sninfr2lnxctl01
      register: node_ready
@@ -63,7 +63,100 @@
        fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
        success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"

-    - name: Settle etcd cluster state
+    - name: Settle etcd cluster consensus
      ansible.builtin.pause:
-        seconds: 10
+        seconds: 15
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+- name: Aerodrome K3s Worker Nodes Sequential Upgrade
+  hosts: k3s_aerodrome_workers
+  gather_facts: false
+  serial: 1
+  vars:
+    target_k3s_version: "v1.31.4+k3s1"
+
+  tasks:
+    - name: Query pre-upgrade worker K3s version
+      ansible.builtin.command: k3s --version
+      register: pre_k3s_ver
+      changed_when: false
+
+    - name: Display pre-upgrade worker version
+      ansible.builtin.debug:
+        msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
+
+    - name: Cordon worker node
+      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      changed_when: true
+
+    - name: Drain worker node
+      ansible.builtin.command: >
+        kubectl drain {{ inventory_hostname }}
+        --ignore-daemonsets
+        --delete-emptydir-data
+        --force
+        --timeout=180s
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      failed_when: false
+      changed_when: true
+
+    - name: Download target K3s release binary on worker
+      ansible.builtin.shell: |
+        set -e
+        ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
+        curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
+        chmod +x /usr/local/bin/k3s.new
+        mv /usr/local/bin/k3s.new /usr/local/bin/k3s
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      register: k3s_download_worker
+
+    - name: Restart K3s Agent systemd service
+      ansible.builtin.systemd:
+        name: k3s-agent
+        state: restarted
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Wait for K3s Agent systemd unit to report active
+      ansible.builtin.command: systemctl is-active k3s-agent
+      register: agent_active
+      until: agent_active.stdout == "active"
+      retries: 25
+      delay: 4
+      changed_when: false
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Uncordon worker node
+      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
+      delegate_to: sninfr2lnxctl01
+      when: target_k3s_version not in pre_k3s_ver.stdout
+      changed_when: true
+
+    - name: Wait for upgraded worker to report Ready in Kubernetes
+      ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      delegate_to: sninfr2lnxctl01
+      register: node_ready
+      until: node_ready.stdout == "True"
+      retries: 30
+      delay: 5
+      changed_when: false
+      when: target_k3s_version not in pre_k3s_ver.stdout
+
+    - name: Query post-upgrade worker K3s version
+      ansible.builtin.command: k3s --version
+      register: post_k3s_ver
+      changed_when: false
+
+    - name: Verify post-upgrade worker version matches target
+      ansible.builtin.assert:
+        that:
+          - target_k3s_version in post_k3s_ver.stdout
+        fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
+        success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
+
+    - name: Settle Longhorn storage replicas between node transitions
+      ansible.builtin.pause:
+        seconds: 15
      when: target_k3s_version not in pre_k3s_ver.stdout
--- a/ansible/playbooks/weekly_os_patching.yml
+++ b/ansible/playbooks/weekly_os_patching.yml
@@ -5,10 +5,46 @@
  gather_facts: true

  tasks:
+    - name: Pre-seed debconf for non-interactive kernel and package upgrades
+      ansible.builtin.shell: |
+        echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
+        echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
+      when: ansible_os_family == "Debian"
+      changed_when: false
+
+    - name: Ensure critical storage prerequisites are installed
+      ansible.builtin.apt:
+        name:
+          - open-iscsi
+          - nfs-common
+        state: present
+        update_cache: false
+      when: ansible_os_family == "Debian" and node_role is defined and node_role in ['master', 'worker']
+
+    - name: Cordon Kubernetes node if applicable
+      ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      changed_when: true
+
+    - name: Drain Kubernetes node if applicable
+      ansible.builtin.command: >
+        kubectl drain {{ inventory_hostname }}
+        --ignore-daemonsets
+        --delete-emptydir-data
+        --force
+        --timeout=180s
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      failed_when: false
+      changed_when: true
+
    - name: Update apt cache
      ansible.builtin.apt:
        update_cache: true
        cache_valid_time: 3600
+      environment:
+        DEBIAN_FRONTEND: noninteractive
      when: ansible_os_family == "Debian"

    - name: Perform dist-upgrade
@@ -16,6 +52,9 @@
        upgrade: dist
        autoremove: true
        autoclean: true
+        dpkg_options: 'force-confdef,force-confold'
+      environment:
+        DEBIAN_FRONTEND: noninteractive
      when: ansible_os_family == "Debian"

    - name: Check if system reboot is required
@@ -28,16 +67,18 @@
      block:
        - name: Reboot host gracefully
          ansible.builtin.reboot:
-            msg: "Automated rolling maintenance reboot via AWX"
+            msg: "Automated rolling maintenance reboot via Snarfnet Automation"
            reboot_timeout: 300
            connect_timeout: 30
            test_command: uptime

-        - name: Verify systemd-timesyncd is active after reboot
-          ansible.builtin.systemd:
-            name: systemd-timesyncd
-            state: started
-          ignore_errors: true
+        - name: Verify systemd-timesyncd or chrony is active after reboot
+          ansible.builtin.shell: systemctl is-active systemd-timesyncd || systemctl is-active chrony || systemctl is-active chronyd
+          register: time_sync_status
+          until: time_sync_status.stdout == "active"
+          retries: 10
+          delay: 5
+          changed_when: false

        - name: Verify zabbix-agent2 is active after reboot
          ansible.builtin.systemd:
@@ -45,3 +86,25 @@
            state: started
          ignore_errors: true

+    - name: Uncordon Kubernetes node if applicable
+      ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      when: node_role is defined and node_role == 'worker'
+      changed_when: true
+
+    - name: Wait for Kubernetes node to report Ready
+      ansible.builtin.command: >
+        kubectl get node {{ inventory_hostname }}
+        -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
+      register: node_ready
+      until: node_ready.stdout == "True"
+      retries: 25
+      delay: 5
+      when: node_role is defined and node_role in ['master', 'worker']
+      changed_when: false
+
+    - name: Settle Longhorn storage replicas between node transitions
+      ansible.builtin.pause:
+        seconds: 15
+      when: node_role is defined and node_role in ['master', 'worker']
--- a/docker-deployments/heimdall.yml
+++ b/docker-deployments/heimdall.yml
@@ -1,20 +1,33 @@
+version: "3.8"
+
 services:
  heimdall:
    image: lscr.io/linuxserver/heimdall:latest
    ports:
-      - "85:80"
-      - "403:443"
+      - target: 80
+        published: 85
+        mode: ingress
+      - target: 443
+        published: 403
+        mode: ingress
    environment:
      - PUID=1000
      - PGID=1000
      - TZ=America/New_York
    volumes:
-      - heimdall_config:/config
+      - type: bind
+        source: /var/lib/docker-data/heimdall/config
+        target: /config
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          - node.hostname == snshpd4lnxswm01
+      restart_policy:
+        condition: on-failure

-volumes:
-  heimdall_config:
-    driver: local
-    driver_opts:
-      type: nfs
-      o: "addr=14.10.10.71,rw,nfsvers=4"
-      device: ":/volume1/docker/heimdall/"
+networks:
+  default:
+    name: homelab_public
+    external: true
--- a/docker-deployments/shepherd.yml
+++ b/docker-deployments/shepherd.yml
@@ -0,0 +1,21 @@
+version: "3.8"
+
+services:
+  shepherd:
+    image: containrrr/shepherd:latest
+    environment:
+      - SLEEP_TIME=24h
+      - IGNORELIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup
+      - UPDATE_OPTIONS=--with-registry-auth
+      - ROLLBACK_FAILURE=true
+      - TZ=America/New_York
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          - node.role == manager
+      restart_policy:
+        condition: on-failure
--- a/docs/DNS_Setup_Plan.md
+++ b/docs/DNS_Setup_Plan.md
@@ -0,0 +1,184 @@
+# Project Blueprint: High-Availability DNS Stack, Network Honeypot & UDM Pro Network Security
+
+## 1. Executive Summary & Live UDM Pro Topology Assessment
+
+Based on the live UDM Pro controller diagnostics, we have mapped the exact network topography, identified critical security exposures, and tailored the implementation plan.
+
+---
+
+### 1.1 Live Network & VLAN Topography
+
+| VLAN ID | Network Name | Subnet / CIDR | Gateway IP | Active Leases | Security Classification |
+|:---:|---|---|---|:---:|---|
+| **1** | **LAN** | `14.44.4.0/25` | `14.44.4.1` | 14 / 117 | Trusted Internal Clients |
+| **2** | **Management** | `140.44.44.0/24` | `140.44.44.1` | 9 / 51 | Out-of-band (iDRAC, Switches, APs) |
+| **3** | **Spacedock** | `140.44.1.0/24` | `140.44.1.1` | 0 / 249 | Harvester VM / Hypervisor Tenant |
+| **20** | **Compute** | `140.44.4.0/25` | `140.44.4.1` | 1 / 15 (Static Nodes) | Mission-Critical K3s Clusters, Swarm |
+| **30** | **Media** | `192.168.1.0/25` | `192.168.1.1` | 1 / 91 | Streaming & Entertainment Devices |
+| **40** | **Storage** | `14.10.10.0/25` | `14.10.10.1` | 2 / 125 | Synology NAS (`.71`, `.72`) NFS / iSCSI |
+| **50** | **Smart Home** | `14.4.4.0/25` | `14.4.4.1` | 53 / 85 | IoT Devices, Gaming Host (`14.4.4.15`) |
+| **60** | **HA** | `140.4.4.0/25` | `140.4.4.1` | 0 / 7 | High-Availability Interconnects |
+| **VPN** | **WireGuard / Teleport** | `192.168.3.0/24` / `.2.0/24` | UDM Pro | Remote | Ingress Remote Access |
+
+---
+
+### 1.2 Critical Security Findings from UDM Pro Diagnostics
+
+1. **Default Security Posture is `Allow All`:**
+   * Currently, the UDM Pro routes traffic freely between all subnets without restriction.
+   * **Risk:** The 53 IoT devices in Smart Home (`14.4.4.0/25`) and Media devices can directly initiate connections to Kubernetes nodes, etcd ports, SSH, and Synology NAS shares in `Compute` (`140.44.4.0/25`) and `Storage` (`14.10.10.0/25`).
+2. **WAN Port Forwarding to Smart Home Host (`14.4.4.15`):**
+   * Port forwards for game servers (`FS25` ports 8443/10823, `Stationeers` ports 6443/27015) point directly from WAN (`174.106.252.96`) to host `14.4.4.15` on the Smart Home VLAN.
+   * If an exploit compromises this host, an attacker currently has unhindered lateral access to the entire homelab infrastructure.
+3. **DNS Leaks & Lack of Local Filtering:**
+   * UDM Pro WAN is set to `Auto DNS Server` (Spectrum ISP DNS `0.0.0.0`).
+   * No DNS ad/malware filtering or split-horizon DNS caching is currently active across the subnets.
+
+---
+
+## 2. High-Availability DNS Stack (Pi-hole + Technitium)
+
+```mermaid
+flowchart TD
+    subgraph Clients["LAN Clients across all VLANs (VLAN 1, 20, 30, 50, VPN)"]
+        Devices["IoT Devices (53), Laptops, Servers, Hypervisors"]
+    end
+
+    subgraph UDM["UniFi Dream Machine Pro (140.44.4.1)"]
+        DHCP["DHCP Option 6 per VLAN:<br>Primary DNS: 140.44.4.35 (Aerodrome VIP)<br>Secondary DNS: 140.44.4.65 (Spaceport VIP)"]
+        NAT["DNS Redirection Rule:<br>Capture port 53 to hardcoded DNS (8.8.8.8) -> Redirect to VIPs"]
+    end
+
+    subgraph AeroCluster["Aerodrome K3s Cluster (VLAN 20)"]
+        AeroVIP["MetalLB VIP: 140.44.4.35:53 (UDP/TCP)"]
+        AeroPihole["Pi-hole Instance 1<br>(Ad-blocking, Caching, FTL rate-limit: 0)"]
+        AeroTech["Technitium Instance 1 (Primary Authoritative)<br>(snarfnet.net Zone, Reverse PTR, DNSSEC)"]
+        AeroStorage["Longhorn Storage Volume"]
+    end
+
+    subgraph SpaceCluster["Spaceport K3s Cluster (VLAN 20)"]
+        SpaceVIP["MetalLB VIP: 140.44.4.65:53 (UDP/TCP)"]
+        SpacePihole["Pi-hole Instance 2<br>(Ad-blocking, Caching, FTL rate-limit: 0)"]
+        SpaceTech["Technitium Instance 2 (Secondary Authoritative)<br>(Zone Transfer AXFR/IXFR from Primary)"]
+        SpaceStorage["Longhorn Storage Volume"]
+    end
+
+    subgraph Upstream["Encrypted Upstream Resolvers"]
+        DoH["DNS-over-HTTPS (Quad9 / Cloudflare / Mullvad)"]
+    end
+
+    Devices -->|DNS Queries (UDP/TCP 53)| DHCP
+    DHCP -->|Primary| AeroVIP
+    DHCP -.->|Secondary / Failover| SpaceVIP
+    AeroVIP --> AeroPihole
+    SpaceVIP --> SpacePihole
+    AeroPihole -->|Recursive / Local Lookups| AeroTech
+    SpacePihole -->|Recursive / Local Lookups| SpaceTech
+    AeroTech <==>|Automatic Zone Sync (AXFR)| SpaceTech
+    AeroTech -->|Encrypted DoH| DoH
+    SpaceTech -->|Encrypted DoH| DoH
+    AeroPihole --- AeroStorage
+    AeroTech --- AeroStorage
+    SpacePihole --- SpaceStorage
+    SpaceTech --- SpaceStorage
+```
+
+### 2.1 Addressing the Rate-Limiting & Sluggishness Issue
+1. **FTL Rate Limit Bypass:** Set `FTLCONF_RATE_LIMIT=0` so local bursts (such as 53 Smart Home devices polling simultaneously) are never throttled.
+2. **Technitium Engine:** Replaces single-threaded Unbound with a high-performance .NET 9 asynchronous resolver capable of servicing tens of thousands of queries per second with sub-millisecond RAM caching.
+3. **High Availability:** MetalLB VIPs on Aerodrome (`140.44.4.35`) and Spaceport (`140.44.4.65`) allow continuous service even during node reboots or maintenance.
+4. **Authoritative Split-Horizon:** Technitium will authoritatively host the internal `snarfnet.net` forward zone and reverse PTR zones (`4.44.140.in-addr.arpa`, `4.4.14.in-addr.arpa`, `10.10.14.in-addr.arpa`), eliminating slow mDNS or external lookup delays.
+
+---
+
+## 3. Network Honeypot & UDM Pro Firewall Hardening
+
+```mermaid
+flowchart LR
+    subgraph WAN_Threats["External & Lateral Threats"]
+        Internet["Public Internet"]
+        CompromisedIoT["Compromised IoT / Host (e.g. 14.4.4.15)"]
+    end
+
+    subgraph UDM_FW["UDM Pro Firewall Rules (LAN IN)"]
+        Rule1["Rule 1: Allow Established & Related"]
+        Rule2["Rule 2: Allow DNS (Port 53 to .35 & .65)"]
+        Rule3["Rule 3: Allow NTP (Port 123 to .11 & .44)"]
+        Rule4["Rule 4: Drop IoT -> Compute (VLAN 50 -> 20)"]
+        Rule5["Rule 5: Drop IoT -> Storage (VLAN 50 -> 40)"]
+        Rule6["Rule 6: Drop Media -> Compute (VLAN 30 -> 20)"]
+    end
+
+    subgraph HoneypotNet["Honeypot Decoy (VLAN 50 / Decoy IP 14.4.4.99)"]
+        OpenCanary["OpenCanary Honeypot<br>(Simulating SSH:22, HTTP:80, SMB:445, RDP:3389)"]
+    end
+
+    subgraph AlertSystem["Alerting & Audit Pipeline"]
+        Gotify["Gotify Push Notification<br>(Instant alert to your phone)"]
+        Zabbix["Zabbix Security Audit Log"]
+    end
+
+    CompromisedIoT -.->|Probes decoy IP| OpenCanary
+    OpenCanary -->|High Priority Webhook| Gotify
+    OpenCanary -->|Syslog Event| Zabbix
+    CompromisedIoT -->|Blocked by Rule 4| UDM_FW
+```
+
+### 3.1 Network Honeypot Deployment
+* **Tool:** **OpenCanary** (lightweight, zero false-positives).
+* **Placement:**
+  * Assigned a decoy IP on the Smart Home subnet (e.g., `14.4.4.99`) where the gaming port forwards exist.
+  * Emulates vulnerable services: SSH (`22`), Telnet (`23`), Web admin (`80`/`8080`), SMB (`445`), and RDP (`3389`).
+* **Alerting:**
+  * When any internal device attempts a port scan or lateral connection to `14.4.4.99`, OpenCanary immediately sends a critical push notification to **Gotify** (`gotify.snarfnet.net`) and generates an alert in **Zabbix**.
+
+---
+
+### 3.2 UDM Pro Firewall Rules Matrix (LAN IN)
+
+We will define firewall groups on the UDM Pro:
+* **RFC1918_Subnets:** All internal private ranges (`140.44.0.0/16`, `14.0.0.0/8`, `192.168.0.0/16`).
+* **K8s_DNS_VIPs:** `140.44.4.35`, `140.44.4.65`.
+* **NTP_Servers:** `140.44.4.11`, `14.4.4.44`.
+
+| Order | Rule Name | Type | Source | Destination | Action | Purpose |
+|:---:|---|:---:|---|---|:---:|---|
+| **2001** | `Allow Established/Related` | LAN IN | Any | Any (State: Established, Related) | **ACCEPT** | Essential return traffic |
+| **2002** | `Allow Cluster DNS` | LAN IN | All VLANs | `K8s_DNS_VIPs` (Port: 53 UDP/TCP) | **ACCEPT** | Clients resolve DNS |
+| **2003** | `Allow Cluster NTP` | LAN IN | All VLANs | `NTP_Servers` (Port: 123 UDP) | **ACCEPT** | Chrony time sync |
+| **2004** | `Allow Trusted to All` | LAN IN | `VLAN 1 (LAN)`, `VLAN 20 (Compute)` | RFC1918_Subnets | **ACCEPT** | Admins & servers access all VLANs |
+| **2005** | `Isolate SmartHome to Compute` | LAN IN | `VLAN 50 (Smart Home)` | `VLAN 20 (Compute)` | **DROP** | Block IoT pivoting into K8s |
+| **2006** | `Isolate SmartHome to Storage` | LAN IN | `VLAN 50 (Smart Home)` | `VLAN 40 (Storage)` | **DROP** | Protect Synology NAS from IoT |
+| **2007** | `Isolate Media to Compute` | LAN IN | `VLAN 30 (Media)` | `VLAN 20 (Compute)` | **DROP** | Block media devices from cluster |
+| **2008** | `Isolate Guest Network` | GUEST IN | Guest Network | RFC1918_Subnets | **DROP** | Restrict guest traffic to Internet only |
+
+---
+
+## 4. Phased Implementation Roadmap
+
+### Phase 1: Teardown Lingering Stubs
+* Cleanly delete orphaned, broken deployments and services in `infra` (Aerodrome) and `infrastructure` (Spaceport):
+  * Remove `deployment/pihole`, `deployment/technitium-dns`, and non-functional NodePort services.
+
+### Phase 2: Deploy Dual-Cluster Pi-hole + Technitium
+* Create Longhorn storage volumes on both clusters for persistent configs and blocklists.
+* Deploy **Aerodrome Primary Stack**:
+  * Technitium on internal port `5353`.
+  * Pi-hole with MetalLB LoadBalancer VIP **`140.44.4.35:53`** (UDP & TCP).
+* Deploy **Spaceport Secondary Stack**:
+  * Technitium on internal port `5353` configured for AXFR zone transfers from Aerodrome.
+  * Pi-hole with MetalLB LoadBalancer VIP **`140.44.4.65:53`** (UDP & TCP).
+* Populate `snarfnet.net` local records and reverse PTR zones.
+
+### Phase 3: Network Honeypot Deployment
+* Deploy containerized OpenCanary with virtual listener on `14.4.4.99`.
+* Configure webhook bridge to Gotify server (`140.44.4.70`) for immediate push notification upon any probe.
+
+### Phase 4: UDM Pro Network Configuration & Policy Hardening
+* Configure DHCP Scope DNS servers on UDM Pro for all VLANs:
+  * Primary: `140.44.4.35`
+  * Secondary: `140.44.4.65`
+* Set UDM Pro WAN DNS to `140.44.4.35` & `140.44.4.65` (or Quad9 fallback).
+* Apply LAN IN firewall rule matrix on UDM Pro to enforce strict VLAN isolation.
+
+snaisvr44hrvstr0x - AppropriateBeav3r39!
--- a/docs/homelab_topology.md
+++ b/docs/homelab_topology.md
@@ -99,12 +99,12 @@ All Kubernetes resources are hosted on the **Compute VLAN (VLAN 20 — `140.44.4

 ---

-### AI Stack (Raspberry Pis)
-*   **Infrastructure role:** Edge AI inference and local web interface.
+### AI Stack (Raspberry Pis with Hailo NPU)
+*   **Infrastructure role:** Edge AI inference backend and Open WebUI frontend.
 *   **SSH Default User:** `snarf`
 *   **IP Allocation:**
-    *   `snnpud4lnxops01` $\rightarrow$ `140.44.4.7` (Raspberry Pi with AI Hat 2, arm64, Ready)
-    *   `snnpud4lnxweb01` $\rightarrow$ `140.44.4.8` (Raspberry Pi running Open WebUI, arm64, Ready)
+    *   `snnpud4lnxops01` $\rightarrow$ `140.44.4.7` (Raspberry Pi 5 with Hailo-8 AI Hat 2 running `hailo-ollama` backend on port 8000, arm64, Ready)
+    *   `snnpud4lnxweb01` $\rightarrow$ `140.44.4.8` (Raspberry Pi running Open WebUI frontend on port 8080, arm64, Ready)

 ---

@@ -133,11 +133,11 @@ This table records how subdomains on `*.snarfnet.net` route to internal IP addre
 | `mesh.snarfnet.net` | HTTPS | `140.44.4.70` (Swarm VIP) | 8086 | MeshCentral Remote Manager |
 | `peanut.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8050 | Peanut Dashboard |
 | `homebox.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 3100 | Homebox Inventory Tracker |
+| `vault.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8085 | VaultConfigurator 3D Modular Storage Planner |
 | `pihole.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8041 | Legacy Swarm Pi-hole 1 |
 | `pihole2.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8042 | Legacy Swarm Pi-hole 2 |
 | `shipyard.snarfnet.net` | HTTPS | `140.44.4.71` (Swarm 01) | 9443 | Portainer Server Console |
-| `shipyardcontrol.snarfnet.net` | HTTPS | `140.44.4.7` (AI Node) | 443 | Turing Pi Management / Hardware Panel |
-| `ai.snarfnet.net` | HTTP | `140.44.4.8` (AI Web Node) | 8080 | Open WebUI Inference Interface |
+| `ai.snarfnet.net` | HTTP | `140.44.4.8` (`snnpud4lnxweb01`) | 8080 | Open WebUI Inference Interface |
 | `aerodrome.snarfnet.net` | HTTPS | `140.44.4.34` (K3s VIP) | 443 | Rancher Management Console |
 | `grafana.snarfnet.net` | HTTP | `140.44.4.32` (K3s VIP) | 88 | Grafana Monitoring Dashboards |
 | `authentik.snarfnet.net` | HTTPS | `140.44.4.29` (K3s VIP) | 444 | Authentik SSO Portal |
--- a/docs/infrastructure_improvement_plan.md
+++ b/docs/infrastructure_improvement_plan.md
@@ -14,7 +14,7 @@ Following Site Reliability Engineering (SRE) best practices, the nine infrastruc
 4. **Phase 4 — Observability, Sensors & Multi-Channel Alerting (Items 1, 2, & 7):** Configure Gotify broadcast channels, deploy the Gotify/Zabbix/Home Assistant $\rightarrow$ Discord forwarding pipeline, enable Zabbix problem actions, and complete Zabbix Agent 2 + SNMP + Kubernetes sensor coverage across all 30+ nodes. *Ensures full visibility before running automated upgrades.* **[100% Completed & Verified]**
 5. **Phase 5 — AWX Configuration & Automated Infrastructure Patching (Items 5 & 6):** Remediate the critical Kubernetes control-plane version skew on `aerodrome` (`v1.26.10` Masters vs. `v1.30.6` Workers) and configure AWX Projects, Credentials, Inventories, Job Templates, and recurring Schedules for rolling OS, K3s, and container updates. **[100% Completed & Verified]**
 6. **Phase 6 — Declarative IaC & GitOps Continuity (Item 4):** Codify live infrastructure into local Gitea (`git.snarfnet.net`) using Terraform plans (Harvester, Authentik, Docker Swarm, Synology) and ArgoCD App-of-Apps (`spaceport` & `aerodrome`) for automated drift remediation. **[100% Completed & Verified]**
-7. **Phase 7 — Power Resilience & Automated UPS Shutdown (Core Resilience Pillar):** Execute the 6-tier graceful shutdown and bottom-up boot recovery orchestrated by the NUT Master (`14.4.4.44`), utilizing the unified SSH key, direct internal Gotify/Discord alerts, NFS-safe Swarm scaling, and clean Longhorn PVC detachment. **[In Progress / Next]**
+7. **Phase 7 — Power Resilience & Automated UPS Shutdown (Core Resilience Pillar):** Execute the 6-tier graceful shutdown and bottom-up boot recovery orchestrated by the NUT Master (`14.4.4.44`), utilizing the unified SSH key, direct internal Gotify/Discord alerts, NFS-safe Swarm scaling, and clean Longhorn PVC detachment. **[100% Completed & Verified]**

 ```mermaid
 flowchart LR
@@ -36,16 +36,16 @@ flowchart LR

 Our automated live probe across the homelab APIs identified the following baseline state that directly informs this plan:

-| Platform / Service | Current Version / State | Key Audit Findings & Action Required |
+| Platform / Service | Current Version / State | Verified Operational Status |
 |---|---|---|
-| **Aerodrome K3s (`local`)** | All 10 Nodes (`ctl01-03` & `wrk01-07`): **`v1.30.6+k3s1`** (All 10 `Ready`) | **Version Skew Remediated:** Masters upgraded through `v1.27.16` $\rightarrow$ `v1.28.15` $\rightarrow$ `v1.29.10` $\rightarrow$ `v1.30.6+k3s1` with zero workload downtime. Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active. |
-| **Spaceport K3s (`c-m-4fgkl75b`)** | 4 Masters + 6 Workers: `v1.31.4+k3s1` (All 10 `Ready`) | Dual default StorageClasses (`local-path` & `longhorn`). 1 PVC stuck `Terminating` (`technitium-longhorn-pvc`). **Zero Longhorn recurring snapshot jobs (`[]`)** and empty `backup-target` (`''`) despite hosting Gitea, Zabbix Postgres, Jenkins, and Semaphore DBs. |
-| **Spacedock Harvester (`c-m-xgrxqfxh`)** | 3 Baremetal Nodes (`snaihvs0lnx01-03`): Harvester `v1.8.0` / `v1.35.2+rke2r1` | Healthy `harvester-longhorn` default StorageClass. 2 active PVCs in `snarfai` (`longhorn-pvc-20tb`, `open-webui-pvc`). Not yet monitored in Zabbix. |
-| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | NPM pinned in `PublishMode: host` on `.74` with daily NAS backup. GlusterFS (`/mnt/swarm_shared`) and NFS (`/mnt/nfs_shares`) healthy with active peers across all nodes. Hardened with `usb_max_current_enable=1`, `tune2fs -e continue`, and `nofail`. |
-| **Zabbix (`zabbix.snarfnet.net`)** | Version `7.0.16` (`140.44.4.57:8888`, Trapper `140.44.4.61:10051`) | 22 hosts registered. **Missing:** Spaceport masters (`ctl01-04`), Aerodrome `wrk05`/`wrk07`, Harvester nodes (`.81-.83`), Synology NAS (`.71`/`.72`), AI Pis (`.7`/`.8`), UDM Pro (`.1`), Home Assistant (`.20`). **Enabled MediaTypes: `[]` (None)**; **Problem Action: Disabled (`status=1`)**. |
-| **Gotify (`gotify.snarfnet.net`)** | Version `2.9.1` (`140.44.4.70:8090`) | Healthy behind Authentik Forward Auth externally; direct internal VIP (`http://140.44.4.70:8090`) reachable for low-level webhooks. |
-| **AWX (`awx.snarfnet.net`)** | Operator `2.19.1` / AWX `24.6.1` (`aerodrome`) | Running in `awx` namespace (`awx-web`, `awx-task`, `awx-postgres-15-0`). Ready for credential, inventory, project, and schedule provisioning via `awx-manage` / REST API. |
-| **ArgoCD (`argo.snarfnet.net`)** | Version `v3.4.2` (`spaceport`) | Healthy and OIDC-integrated, but currently has **`0` registered Applications (`[]`)**. |
+| **Aerodrome K3s (`local`)** | All 10 Nodes (`ctl01-03` & `wrk01-07`): **`v1.30.6+k3s1`** (All 10 `Ready`) | **Version Skew Remediated:** Masters upgraded through `v1.27.16` $\rightarrow$ `v1.28.15` $\rightarrow$ `v1.29.10` $\rightarrow$ `v1.30.6+k3s1` with zero workload downtime. Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active. Enrolled in ArgoCD. |
+| **Spaceport K3s (`c-m-4fgkl75b`)** | 4 Masters + 6 Workers: **`v1.31.4+k3s1`** (All 10 `Ready`) | Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active across Gitea, Zabbix Postgres, Jenkins, and Semaphore DBs. ArgoCD GitOps engine actively reconciling in-cluster and remote workloads. |
+| **Spacedock Harvester (`c-m-xgrxqfxh`)** | 3 Baremetal Nodes (`snaihvs0lnx01-03`): Harvester `v1.8.0` / `v1.35.2+rke2r1` | Healthy `harvester-longhorn` default StorageClass. Declarative VLAN networks and VM templates codified in Terraform. Monitored via SNMP / ICMP. |
+| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | Dual-NPM HA Keepalived cluster active on VIP `140.44.4.70` across `snshpd4lnxswm01` (.71) and `snshpd4lnxswm02` (.72) in `PublishMode: host`. Hardened NFS persistent mounts (`/mnt/nfs_shares`) on Synology NAS (`14.10.10.71:/volume1/docker`). Automated SQLite backup timer active with daily tarball sync to NAS. |
+| **Zabbix (`zabbix.snarfnet.net`)** | Version `7.0.16` (`140.44.4.57:8888`, Trapper `140.44.4.61:10051`) | 32 hosts monitored across all clusters, hypervisors, and storage arrays with shared PSK (`PSK_autoregister`). Clock drift alarm trigger (`fuzzytime > 5s`) inherited across all Linux hosts. Gotify & Discord problem actions active. |
+| **Gotify (`gotify.snarfnet.net`)** | Version `2.9.1` (`140.44.4.70:8090`) | 5 structured notification channels provisioned. Internal VIP (`http://140.44.4.70:8090`) active for direct programmatic alerts. `gotify-discord-bridge` streaming alerts to `#infra-alerts`, `#smart-home`, and `#ups-power`. |
+| **AWX (`awx.snarfnet.net`)** | Operator `2.19.1` / AWX `24.6.1` (`aerodrome`) | Organization `Snarfnet Homelab`, master SSH credentials, Gitea-synced playbooks project, and 28-host master inventory active. Daily drift audit and rolling upgrade job templates verified green. |
+| **ArgoCD (`argo.snarfnet.net`)** | Version `v3.4.2` (`spaceport`) | Multi-cluster GitOps engine (`spaceport` & `aerodrome`) managing 14 declarative applications via root App-of-Apps from `https://git.snarfnet.net/Snarf/homelab-gitops.git`. Real-time notifications routed to Gotify & Discord. |

 ---

@@ -374,7 +374,23 @@ flowchart TD

 ---

-### Phase 7: Power Resilience & UPS Automation Plan (Updated Core Pillar)
+### Phase 7: Power Resilience & UPS Automation Plan (Updated Core Pillar) — [100% Completed & Verified]
+
+#### Architecture & Implementation Steps (Executed & Verified)
+1. **NUT Master Server Configuration (`14.4.4.44`):**
+   - Configured `upsmon.conf` on `nutserver` with `NOTIFYCMD /usr/local/bin/nut-notify.sh` and `SHUTDOWNCMD /usr/local/bin/homelab_power_shutdown.sh`.
+   - Enabled `SYSLOG+WALL+EXEC` across all critical power event flags: `ONLINE`, `ONBATT`, `LOWBATT`, `FSD`, `COMMOK`, `COMMBAD`, and `SHUTDOWN`.
+   - Verified `nut-monitor.service` is active, connected to `server-closet@localhost (primary)`.
+2. **Notification Pipeline Verification (`scripts/nut_notify.sh`):**
+   - Verified automated dispatch to Gotify VIP (`http://140.44.4.70:8090/message`) with token `A4uN4fEmErGeNcYtOkEn` (Application 1: `UPS & Power Resilience`, priority 9/10).
+   - Verified automatic streaming to Discord channel `#ups-power` via WebSocket bridge, with direct UDM-Pro egress fallback if Gotify/Swarm is stopped.
+3. **Graceful 6-Tier Shutdown Automation (`scripts/homelab_power_shutdown.sh`):**
+   - Tier 1: Stops transient AI inference Pis (`140.44.4.7`, `140.44.4.8`).
+   - Tier 2: Scales down Swarm NFS services (`nginx-proxy-manager_app`, `backup`, `peanut`, `homebox`) on `140.44.4.71` to cleanly flush databases and release NFS file locks.
+   - Tier 3: Stops K3s agents on all 7 Aerodrome workers (`140.44.4.21-27`) and 6 Spaceport workers (`140.44.4.51-56`), cleanly syncing and detaching Longhorn CSI persistent volumes.
+   - Tier 4: Signals Synology NAS arrays (`14.10.10.71`, `14.10.10.72`) into DSM Safe Mode via native NUT slave daemon (`synopoweroff`).
+   - Tier 5: Saves on-demand pre-shutdown etcd snapshots on Spaceport masters (`.41-.44`) and Aerodrome masters (`.11-.13`), stops K3s service, and halts master nodes and Swarm managers (`.71-.75`).
+   - Tier 6: NUT Master fires final FSD notification, arms `/etc/killpower`, and executes `/sbin/shutdown -h +0`.

 #### 6.1 Core Power Architecture & IP Assignments

@@ -383,7 +399,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
 | Role | Hostname / Identifier | Target IP | Subnet / VLAN | Description & Failure Protocol |
 |---|---|---|---|---|
 | **NUT Master Server** | `nutserver.snarfnet.net` | `14.4.4.44` | VLAN 50 (Smart Home) | Primary NUT `upsd` daemon polling physical UPS via USB/Serial/SNMP. Runs `upsmon master` and holds `/root/.ssh/id_ed25519_snarfnet`. |
-| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.74` | VLAN 20 (Swarm 04) | NPM running in `PublishMode: host` on ports 80/443/81. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
+| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.70` | VLAN 20 (Keepalived VIP) | Dual-NPM active/standby running in `PublishMode: host` on ports 80/443/81 across Swarm 01 & 02. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
 | **Authentication & SSO** | `authentik.snarfnet.net` | `140.44.4.29:444` | VLAN 20 (Aerodrome K3s) | Handles Forward Auth & Conditional MFA. Runs on Longhorn storage in Aerodrome. |
 | **UPS Web Dashboard** | `peanut.snarfnet.net` | `140.44.4.70:8050` | VLAN 20 (Swarm VIP) | Peanut container UI consuming `14.4.4.44:3493`. Protected by Authentik Forward Auth externally. |
 | **Emergency Push & Discord** | `gotify.snarfnet.net` | `140.44.4.70:8090` | VLAN 20 (Swarm VIP) | Gotify server + `gotify-discord-bridge`. **Internal outage alerts use `http://140.44.4.70:8090`** (+ direct Discord webhook fallback) to bypass Authentik during shutdown. |
@@ -450,7 +466,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
 3. **Stage 3 (NUT Master):** `14.4.4.44` boots (`BIOS: Restore on AC Power Loss = Always On`), starts `upsd`, and sends `ONLINE` alert to Discord/Gotify.
 4. **Stage 4 (Storage Gate — Must Complete Before Compute):** Synology NAS arrays (`14.10.10.71` & `14.10.10.72`) boot, mount Btrfs pools, and expose NFS (`:2049`).
 5. **Stage 5 (Control Planes & Hypervisors):** Harvester (`140.44.4.81-83`), K3s Masters (`aerodrome` & `spaceport`), and Swarm Managers (`140.44.4.71-75`) boot and establish etcd/Raft quorum.
-6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and NPM on `snshpd4lnxswm04` (`140.44.4.74`) binds host ports `80/443/81`.
+6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and Dual-NPM on Swarm 01 & 02 (`140.44.4.71-72`) binds host ports `80/443/81` behind Keepalived VIP `140.44.4.70`.

 ---

--- a/docs/opencanary_honeypot.md
+++ b/docs/opencanary_honeypot.md
@@ -0,0 +1,59 @@
+# OpenCanary Decoy Honeypot Architecture & Configuration
+
+## 1. Overview
+As part of the **Snarfnet Network Security Hardening Blueprint (Phase 3)**, an **OpenCanary Honeypot** has been deployed to detect lateral movement, unauthorized reconnaissance, and compromised devices inside the **Smart Home VLAN (VLAN 50, `14.4.4.0/25`)**.
+
+* **Decoy IP:** `14.4.4.99` (Subnet: `14.4.4.0/25`, Gateway: `14.4.4.1`)
+* **Host Machine:** `snshpd4lnxswm01` (`140.44.4.71`)
+* **Physical Uplink:** Direct 802.1Q trunk port with tagged VLAN 50 support.
+* **Network Mode:** Native Docker `macvlan` attached to subinterface `eth0.50`.
+
+---
+
+## 2. Emulated Services & Ports
+
+The honeypot presents realistic service banners designed to trap automated scanners and unauthorized intruders without exposing real systems:
+
+| Port | Protocol | Emulated Service | Decoy Profile |
+|:---:|:---:|:---|:---|
+| **21** | TCP | **FTP** | Linux FTP server (`FTP server ready`) |
+| **22** | TCP | **SSH** | Ubuntu OpenSSH banner (`SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.10`) |
+| **23** | TCP | **Telnet** | Ubuntu login prompt (`Ubuntu 22.04 LTS login: `) |
+| **80** | TCP | **HTTP Web** | Synology/NAS Web Admin Login skin |
+| **1433** | TCP | **MSSQL** | Microsoft SQL Server 2012 |
+| **3306** | TCP | **MySQL** | MySQL 5.7 database server |
+| **3389** | TCP | **RDP** | Windows Remote Desktop Protocol listener |
+| **5000** | TCP | **VNC** | Remote framebuffer / VNC listener |
+| **8080** | TCP | **HTTP Proxy** | Squid HTTP proxy decoy |
+| **9418** | TCP | **Git** | Git daemon repository service |
+
+---
+
+## 3. Real-Time Alerting Pipeline
+
+```mermaid
+flowchart LR
+    Attacker["Suspicious Host / IoT Device<br>(e.g. 14.4.4.15)"] -->|TCP / Port Probe| Decoy["OpenCanary (14.4.4.99:22/80/445/3389)"]
+    Decoy -->|Structured JSON Log| LogFile["/opt/opencanary/logs/opencanary.log"]
+    LogFile -->|Real-Time Tail| Watcher["opencanary-watcher.service<br>(Python Daemon)"]
+    Watcher -->|HTTP POST (Priority 8)| Gotify["Gotify Server (App 6)<br>(gotify.snarfnet.net)"]
+    Gotify -->|Instant Push Notification| Phone["Admin Mobile App"]
+    Gotify -->|Discord Webhook Bridge| Discord["Discord #infra-alerts"]
+```
+
+1. Any TCP connection, banner grab, authentication attempt, or probe arriving on `14.4.4.99` generates a structured event in `/opt/opencanary/logs/opencanary.log`.
+2. The `opencanary-watcher.service` daemon ingests the event in real-time, extracts the attacker's source IP, source port, targeted port, service type, and credentials (if entered).
+3. The event is posted to **Gotify Application #6** (`A9oPeNcAnArYhOnEyPoT`) with high priority (`8`).
+4. `gotify-discord-bridge` formats a rich embed and posts the alert directly into the Discord `#infra-alerts` channel.
+
+---
+
+## 4. Systemd Services & Maintenance
+
+* **VLAN Subinterface Service:**
+  `/etc/systemd/system/opencanary-vlan.service` (Ensures `eth0.50` is brought up on boot).
+* **Alert Watcher Service:**
+  `/etc/systemd/system/opencanary-watcher.service` (Tails log file and delivers push alerts).
+* **Docker Container:**
+  `opencanary` (Managed via Docker with `restart: unless-stopped` on network `opencanary_net`).
+
--- a/docs/snarfnet_homelab_topology.html
+++ b/docs/snarfnet_homelab_topology.html
--- a/docs/snarfnet_homelab_topology.png
+++ b/docs/snarfnet_homelab_topology.png
--- a/2
+++ b/2
--- a/scripts/add_forward_rules.py
+++ b/scripts/add_forward_rules.py
@@ -0,0 +1,22 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f">>> {cmd}")
+    _, o, e = ssh.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    if out:
+        print(out)
+    if err:
+        print("[stderr]", err)
+
+run("iptables -I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT")
+run("iptables -I FORWARD 2 -p tcp -d 140.44.4.64 --dport 10051 -j ACCEPT")
+run("iptables -L FORWARD -n -v | head -n 8")
+
+ssh.close()
+
--- a/scripts/add_kuma_bookmarks.py
+++ b/scripts/add_kuma_bookmarks.py
@@ -0,0 +1,58 @@
+import yaml
+
+def str_presenter(dumper, data):
+    if '\n' in data:
+        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+
+yaml.add_representer(str, str_presenter)
+yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+docs = list(yaml.safe_load_all(text))
+cm = docs[2]
+
+bookmarks_yaml = """- Quick Links:
+    - GitHub:
+        - abbr: GH
+          href: https://github.com/
+    - UniFi Site Manager:
+        - abbr: UI
+          href: https://unifi.ui.com/
+    - Cloudflare:
+        - abbr: CF
+          href: https://dash.cloudflare.com/
+- Grafana Telemetry:
+    - Systems Performance:
+        - abbr: SYS
+          href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861
+    - Container Workloads:
+        - abbr: K8S
+          href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads
+    - AI Stack Intelligence:
+        - abbr: AI
+          href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence
+- Uptime Status Pages:
+    - Unified Overview:
+        - abbr: ALL
+          href: https://uptime.snarfnet.net/status/default
+    - Core & VIP Fabric:
+        - abbr: VIP
+          href: https://uptime.snarfnet.net/status/core
+    - GitOps & CI/CD:
+        - abbr: OPS
+          href: https://uptime.snarfnet.net/status/gitops
+    - Edge & Ingress:
+        - abbr: EDG
+          href: https://uptime.snarfnet.net/status/edge
+"""
+
+cm['data']['bookmarks.yaml'] = bookmarks_yaml
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
+    yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+print("Updated bookmarks in homepage.yaml successfully!")
+
--- a/scripts/add_media_to_snarf.py
+++ b/scripts/add_media_to_snarf.py
@@ -0,0 +1,32 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
+
+payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
+token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+media_payload = {
+    'jsonrpc': '2.0',
+    'method': 'user.update',
+    'params': {
+        'userid': '3',
+        'medias': [
+            {
+                'mediatypeid': '70',
+                'sendto': 'snarf@snarfnet.net',
+                'active': 0,
+                'severity': 63, # 1+2+4+8+16+32
+                'period': '1-7,00:00-24:00'
+            }
+        ]
+    },
+    'id': 2
+}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(media_payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
+res = json.loads(urllib.request.urlopen(req, context=ctx).read())
+print("User update result:", res)
--- a/scripts/add_topology_to_homepage.py
+++ b/scripts/add_topology_to_homepage.py
@@ -0,0 +1,26 @@
+import yaml
+
+def str_presenter(dumper, data):
+    if '\n' in data:
+        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+
+yaml.add_representer(str, str_presenter)
+yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+docs = list(yaml.safe_load_all(text))
+cm = docs[2]
+
+services = yaml.safe_load(cm['data']['services.yaml'])
+bookmarks = yaml.safe_load(cm['data']['bookmarks.yaml'])
+
+cm['data']['services.yaml'] = yaml.dump(services, sort_keys=False, default_flow_style=False, allow_unicode=True)
+cm['data']['bookmarks.yaml'] = yaml.dump(bookmarks, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
+    yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+print("Cleaned up unicode strings in homepage.yaml!")
--- a/scripts/add_zabbix_devices.py
+++ b/scripts/add_zabbix_devices.py
@@ -0,0 +1,118 @@
+import json
+import ssl
+import urllib.request
+
+ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
+ZABBIX_USER = "Admin"
+ZABBIX_PASS = "zabbix"
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def zbx_call(method, params, auth=None):
+    payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
+    headers = {"Content-Type": "application/json-rpc"}
+    if auth:
+        headers["Authorization"] = f"Bearer {auth}"
+    req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
+    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+        resp = json.loads(r.read().decode("utf-8"))
+        if "error" in resp:
+            raise RuntimeError(f"Zabbix error in {method}: {resp['error']}")
+        return resp["result"]
+
+auth = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS})
+
+# Check existing host names
+existing_hosts = zbx_call("host.get", {"output": ["hostid", "host", "name"]}, auth)
+existing_names = {h["host"].lower(): h["hostid"] for h in existing_hosts}
+
+# Create host groups if not exist
+groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth)
+group_map = {g["name"]: g["groupid"] for g in groups}
+
+for gname in ["Network Devices", "Environmental & Sensors", "Management & IPMI"]:
+    if gname not in group_map:
+        res = zbx_call("hostgroup.create", {"name": gname}, auth)
+        group_map[gname] = res["groupids"][0]
+        print(f"Created Host Group: {gname} (ID: {group_map[gname]})")
+
+devices_to_add = [
+    {
+        "host": "idrac-spacedock4",
+        "name": "Dell PowerEdge R730 Node 4 iDRAC (idrac-spacedock4)",
+        "ip": "140.44.44.124",
+        "group": "Management & IPMI"
+    },
+    {
+        "host": "usw-agg-core",
+        "name": "UniFi Switch Aggregation 10G Core",
+        "ip": "14.44.4.26",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-24-poe-closet",
+        "name": "UniFi Switch 24 PoE Closet",
+        "ip": "14.44.4.61",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-pro-24-poe-compute",
+        "name": "UniFi Switch Pro 24 PoE Compute",
+        "ip": "14.44.4.107",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-flex-attic",
+        "name": "UniFi Switch Flex Attic",
+        "ip": "14.44.4.98",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-flex-garage",
+        "name": "UniFi Switch Flex Garage",
+        "ip": "14.44.4.96",
+        "group": "Network Devices"
+    },
+    {
+        "host": "tempest-weather-station",
+        "name": "Tempest Weather Station Gateway",
+        "ip": "14.4.4.47",
+        "group": "Environmental & Sensors"
+    },
+    {
+        "host": "sense-energy-monitor",
+        "name": "Sense Home Energy Gateway",
+        "ip": "14.4.4.48",
+        "group": "Environmental & Sensors"
+    }
+]
+
+for dev in devices_to_add:
+    hname = dev["host"]
+    if hname.lower() in existing_names:
+        print(f"[EXISTS] {hname}")
+        continue
+    params = {
+        "host": dev["host"],
+        "name": dev["name"],
+        "interfaces": [
+            {
+                "type": 1, # Agent / ICMP
+                "main": 1,
+                "useip": 1,
+                "ip": dev["ip"],
+                "dns": "",
+                "port": "10050"
+            }
+        ],
+        "groups": [{"groupid": group_map[dev["group"]]}],
+        "templates": [{"templateid": "10564"}] # ICMP Ping
+    }
+    try:
+        res = zbx_call("host.create", params, auth)
+        print(f"[CREATED] {dev['name']} ({dev['ip']}) -> HostID {res['hostids'][0]}")
+    except Exception as e:
+        print(f"[FAILED] {dev['name']}: {e}")
+
--- a/scripts/apply_36.py
+++ b/scripts/apply_36.py
@@ -0,0 +1,65 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n dns delete svc pihole-dns-vip pihole-web-vip --wait=false
+sleep 2
+cat << 'EOF' | k3s kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: pihole-dns-vip
+  namespace: dns
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 140.44.4.36
+    metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
+  labels:
+    app: pihole
+spec:
+  type: LoadBalancer
+  selector:
+    app: pihole
+  ports:
+  - name: dns-udp
+    port: 53
+    targetPort: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    targetPort: 53
+    protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pihole-web-vip
+  namespace: dns
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 140.44.4.36
+    metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
+  labels:
+    app: pihole
+spec:
+  type: LoadBalancer
+  selector:
+    app: pihole
+  ports:
+  - name: web-http
+    port: 80
+    targetPort: 80
+    protocol: TCP
+EOF
+sleep 3
+k3s kubectl -n dns get svc
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode('utf-8'))
+c.close()
+
--- a/scripts/apply_and_set_pihole_pw.py
+++ b/scripts/apply_and_set_pihole_pw.py
@@ -0,0 +1,52 @@
+import paramiko
+import time
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+def apply_and_update(ip, name, manifest_path):
+    print(f"\n=======================================================")
+    print(f"Applying to {name} ({ip})...")
+    with open(manifest_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=key_path, timeout=10)
+
+    stdin, stdout, stderr = ssh.exec_command("k3s kubectl apply -f -")
+    stdin.write(content)
+    stdin.channel.shutdown_write()
+    print("Apply output:", stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("Apply stderr:", err)
+
+    print(f"Waiting for Pi-hole deployment rollout on {name}...")
+    stdin, stdout, stderr = ssh.exec_command("k3s kubectl rollout status deployment/pihole -n dns --timeout=60s")
+    print(stdout.read().decode())
+
+    # Get new pod name
+    stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -l app=pihole -o jsonpath='{.items[0].metadata.name}'")
+    pod_name = stdout.read().decode().strip()
+    print(f"New Pi-hole pod: {pod_name}")
+
+    # Set password to snarfnetadmin
+    print("Setting Pi-hole password to 'snarfnetadmin'...")
+    stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod_name} -- pihole setpassword snarfnetadmin")
+    out = stdout.read().decode()
+    err = stderr.read().decode()
+    print("Set password output:", out)
+    if err:
+        print("Set password stderr:", err)
+
+    ssh.close()
+
+aero_manifest = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml"
+space_manifest = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml"
+
+apply_and_update("140.44.4.11", "Aerodrome", aero_manifest)
+apply_and_update("140.44.4.41", "Spaceport", space_manifest)
+print("\nDone!")
+
--- a/scripts/apply_bios_tuning.py
+++ b/scripts/apply_bios_tuning.py
@@ -0,0 +1,51 @@
+import paramiko
+import time
+
+targets = [
+    {
+        "name": "Node 1 (140.44.44.121)",
+        "ip": "140.44.44.121",
+        "settings": [
+            ("bios.IntegratedDevices.SriovGlobalEnable", "Enabled")
+        ]
+    },
+    {
+        "name": "Node 4 (140.44.44.124)",
+        "ip": "140.44.44.124",
+        "settings": [
+            ("bios.SysProfileSettings.SysProfile", "PerfOptimized")
+        ]
+    }
+]
+
+for t in targets:
+    print(f"\n==========================================")
+    print(f"Applying BIOS settings to {t['name']}...")
+    try:
+        c = paramiko.SSHClient()
+        c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        c.connect(t['ip'], username='root', password='Katzir476!', timeout=15)
+        
+        for key, val in t['settings']:
+            cmd = f"racadm set {key} {val}"
+            stdin, stdout, stderr = c.exec_command(cmd)
+            out = stdout.read().decode().strip()
+            print(f"  racadm set {key} {val}:\n    {out}")
+            time.sleep(1)
+
+        # Create BIOS job
+        stdin, stdout, stderr = c.exec_command("racadm jobqueue create BIOS.Setup.1-1")
+        job_out = stdout.read().decode().strip()
+        print(f"  racadm jobqueue create BIOS.Setup.1-1:\n    {job_out}")
+        
+        # View job queue
+        stdin, stdout, stderr = c.exec_command("racadm jobqueue view")
+        q_out = stdout.read().decode().strip()
+        print(f"  Job Queue:\n    {q_out[:200]}")
+        
+        c.close()
+    except Exception as e:
+        print(f"  Error on {t['name']}: {e}")
+
+print("\nDone applying BIOS settings!")
+
--- a/scripts/apply_coredns_spaceport.py
+++ b/scripts/apply_coredns_spaceport.py
@@ -0,0 +1,48 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+
+cm_yaml = """cat << 'EOF' | k3s kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  snarfnet.override: |
+    hosts {
+      140.44.4.34 aerodrome.snarfnet.net
+      140.44.4.90 harvester.spacedock.snarfnet.net
+      fallthrough
+    }
+EOF
+"""
+
+print("Applying coredns-custom on Spaceport...")
+stdin, stdout, stderr = client.exec_command(cm_yaml)
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+print("\nRestarting coredns deployment on Spaceport...")
+client.exec_command("k3s kubectl -n kube-system rollout restart deployment/coredns")
+client.exec_command("k3s kubectl -n kube-system rollout status deployment/coredns --timeout=60s")
+
+print("\nRestarting cattle-cluster-agent on Spaceport...")
+client.exec_command("k3s kubectl -n cattle-system rollout restart deployment/cattle-cluster-agent")
+client.exec_command("k3s kubectl -n cattle-system rollout status deployment/cattle-cluster-agent --timeout=60s")
+
+time.sleep(5)
+print("\nChecking cattle-cluster-agent logs:")
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent --tail=25")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+
--- a/scripts/apply_dns_updates.py
+++ b/scripts/apply_dns_updates.py
@@ -0,0 +1,31 @@
+import paramiko
+
+SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def apply_manifest(host, local_file):
+    with open(local_file, "r") as f:
+        manifest_content = f.read()
+    
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(host, username="root", key_filename=SSH_KEY, timeout=10)
+    
+    stdin, stdout, stderr = client.exec_command("k3s kubectl apply -f -")
+    stdin.write(manifest_content)
+    stdin.close()
+    out = stdout.read().decode()
+    err = stderr.read().decode()
+    client.close()
+    return out, err
+
+print("=== Applying to Aerodrome (140.44.4.11) ===")
+out, err = apply_manifest("140.44.4.11", r"homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml")
+print(out)
+if err:
+    print("Stderr:", err)
+
+print("=== Applying to Spaceport (140.44.4.41) ===")
+out, err = apply_manifest("140.44.4.41", r"homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml")
+print(out)
+if err:
+    print("Stderr:", err)
--- a/scripts/apply_full_homepage.py
+++ b/scripts/apply_full_homepage.py
@@ -0,0 +1,250 @@
+import yaml
+
+def str_presenter(dumper, data):
+    if '\n' in data:
+        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+
+yaml.add_representer(str, str_presenter)
+yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+docs = list(yaml.safe_load_all(text))
+cm = docs[2]
+
+services_yaml = """- Critical Services:
+    - Authentik:
+        icon: authentik.png
+        href: https://authentik.snarfnet.net
+        description: SSO & Identity Management
+    - PeaNUT UPS:
+        icon: peanut.png
+        href: https://portal.snarfnet.net
+        description: UPS Power & Battery Monitor
+    - Unifi Controller:
+        icon: unifi.png
+        href: https://unifi.ui.com
+        widget:
+          type: unifi
+          url: https://140.44.44.1
+          username: Snarf
+          password: Katzir476!!!
+
+- GitOps & Automation:
+    - Gitea:
+        icon: gitea.png
+        href: https://git.snarfnet.net
+        description: Git Repositories
+        widget:
+          type: gitea
+          url: https://git.snarfnet.net
+          key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71
+    - ArgoCD:
+        icon: argocd.png
+        href: https://argo.snarfnet.net
+        description: Declarative GitOps Reconciler
+        widget:
+          type: argocd
+          url: https://argo.snarfnet.net
+          key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0
+    - AWX Automation:
+        icon: awx.png
+        href: https://awx.snarfnet.net
+        description: Ansible Automation Controller
+    - Semaphore:
+        icon: semaphore.png
+        href: https://semaphore.snarfnet.net
+        description: Playbook Execution Engine
+
+- Storage & Clusters:
+    - Longhorn Aerodrome:
+        icon: longhorn.png
+        href: https://longhorn.aerodrome.snarfnet.net
+        description: Aerodrome Distributed Flash Storage
+    - Longhorn Spaceport:
+        icon: longhorn.png
+        href: https://longhorn.spaceport.snarfnet.net
+        description: Spaceport Distributed Block Storage
+    - Rancher (Aerodrome):
+        icon: rancher.png
+        href: https://aerodrome.snarfnet.net
+        description: Kubernetes Management Console
+        widget:
+          type: customapi
+          url: https://aerodrome.snarfnet.net/v3/clusters/local
+          headers:
+            Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
+          mappings:
+            - field: capacity.cpu
+              label: CPU Cores
+            - field: capacity.pods
+              label: Max Pods
+            - field: state
+              label: Cluster State
+    - Harvester HCI (Spacedock):
+        icon: harvester.png
+        href: https://harvester.spacedock.snarfnet.net
+        description: HCI Virtualization Hypervisors
+        widget:
+          type: customapi
+          url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt
+          headers:
+            Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
+          mappings:
+            - field: capacity.cpu
+              label: CPU Cores
+            - field: capacity.pods
+              label: Max Pods
+            - field: state
+              label: HCI State
+    - Portainer Swarm:
+        icon: portainer.png
+        href: https://shipyard.snarfnet.net
+        description: Docker Swarm Container Fleet
+        widget:
+          type: portainer
+          url: https://140.44.4.71:9443
+          env: 3
+          key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8=
+
+- Infrastructure & Observability:
+    - NGINX Proxy Manager:
+        icon: nginx-proxy-manager
+        href: https://nginx.snarfnet.net
+        description: Reverse Proxy & SSL Certificates
+    - Uptime Kuma:
+        icon: uptime-kuma
+        href: https://uptime.snarfnet.net
+        description: Multi-Endpoint Availability Probes
+        widget:
+          type: uptimekuma
+          url: http://140.44.4.28:448
+          slug: default
+    - Zabbix:
+        icon: zabbix.png
+        href: https://zabbix.snarfnet.net
+        description: Infrastructure Metrics & Alarms
+    - Grafana:
+        icon: grafana.png
+        href: https://grafana.snarfnet.net
+        description: Real-Time Metrics & AI Dashboards
+        widget:
+          type: grafana
+          url: https://grafana.snarfnet.net
+          username: admin
+          password: Katzir476!!!
+    - Core Node Glances:
+        icon: glances.png
+        href: http://140.44.4.11:61208
+        description: Host Resource Monitor
+
+- Environment & Weather:
+    - Tempest Weather Station:
+        icon: weather.png
+        href: https://snarfhome.snarfnet.net
+        description: Backyard Weather Telemetry
+        widget:
+          type: homeassistant
+          url: http://14.4.4.20:8123
+          key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4
+          custom:
+            - template: "{{ states('sensor.st_00158555_temperature') | float(0) | round(1) }} °F"
+              label: Outdoor Temp
+            - template: "{{ states('sensor.st_00158555_feels_like') | float(0) | round(1) }} °F"
+              label: Feels Like
+            - template: "{{ states('sensor.st_00158555_humidity') | float(0) | round(1) }} %"
+              label: Humidity
+            - template: "{{ states('sensor.st_00158555_wind_speed') | float(0) | round(1) }} mph"
+              label: Wind Speed
+"""
+
+settings_yaml = """title: SnarfNet Operations & Telemetry
+headerStyle: boxed
+color: slate
+theme: dark
+statusStyle: badge
+background:
+  image: https://i.imgur.com/hOzzkCY.jpeg
+  blur: sm
+  opacity: 50
+  brightness: 50
+cardBlur: sm
+quicklaunch:
+  searchDescriptions: true
+  hideInternetSearch: true
+  hideVisitURL: true
+  hideVersion: true
+layout:
+  Critical Services:
+    icon: mdi-shield-check
+    style: row
+    columns: 3
+  GitOps & Automation:
+    icon: mdi-git
+    style: row
+    columns: 4
+  Storage & Clusters:
+    icon: mdi-server-network
+    style: row
+    columns: 3
+  Infrastructure & Observability:
+    icon: mdi-chart-timeline-variant
+    style: row
+    columns: 3
+  Environment & Weather:
+    icon: mdi-weather-partly-cloudy
+    style: row
+    columns: 1
+showStats: true
+providers:
+  longhorn:
+    url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500
+"""
+
+widgets_yaml = """- kubernetes:
+    cluster:
+      show: true
+      cpu: true
+      memory: true
+      show_names: true
+    nodes:
+      show: false
+- longhorn:
+    total: true
+    nodes: false
+- resources:
+    label: System
+    backend: resources
+    cpu: true
+    expanded: true
+    memory: true
+    disk: /
+- search:
+    provider: duckduckgo
+    target: _blank
+"""
+
+bookmarks_yaml = """- Quick Links:
+    - GitHub:
+        - abbr: GH
+          href: https://github.com/
+    - UniFi Site Manager:
+        - abbr: UI
+          href: https://unifi.ui.com/
+    - Cloudflare:
+        - abbr: CF
+          href: https://dash.cloudflare.com/
+"""
+
+cm['data']['services.yaml'] = services_yaml
+cm['data']['settings.yaml'] = settings_yaml
+cm['data']['widgets.yaml'] = widgets_yaml
+cm['data']['bookmarks.yaml'] = bookmarks_yaml
+cm['data']['kubernetes.yaml'] = "mode: cluster\n"
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
+    yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+print("Saved clean homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml successfully!")
--- a/scripts/apply_grafana_dashboards.py
+++ b/scripts/apply_grafana_dashboards.py
@@ -0,0 +1,33 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r", encoding="utf-8") as f:
+    grafana_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_updated.yaml")
+stdin.write(grafana_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying Grafana manifests:")
+print(run("kubectl apply -f /tmp/grafana_updated.yaml"))
+
+print("\nRestarting Grafana:")
+print(run("kubectl rollout restart deployment/grafana -n default"))
+print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
+
+print("\nWaiting for Grafana to finish initialization...")
+time.sleep(10)
+
+c.close()
+
--- a/scripts/apply_grafana_server_side.py
+++ b/scripts/apply_grafana_server_side.py
@@ -0,0 +1,20 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+print("Applying server-side:")
+print(run("kubectl apply -f /tmp/grafana_full.yaml --server-side --force-conflicts"))
+print("\nRollout restart:")
+print(run("kubectl rollout restart deployment/grafana -n default"))
+print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
+
+c.close()
+
--- a/scripts/apply_h21_authentik.py
+++ b/scripts/apply_h21_authentik.py
@@ -0,0 +1,149 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+adv_config = """# Buffer tuning for Authentik headers
+proxy_buffers 8 16k;
+proxy_buffer_size 32k;
+
+# Outpost verification route - plain HTTP to internal VIP
+location /outpost.goauthentik.io {
+    auth_request off;
+    proxy_pass              http://140.44.4.29:9000/outpost.goauthentik.io;
+    proxy_set_header        Host authentik.snarfnet.net;
+    proxy_set_header        X-Original-URL $scheme://$http_host$request_uri;
+    add_header              Set-Cookie $auth_cookie;
+    auth_request_set        $auth_cookie $upstream_http_set_cookie;
+    proxy_pass_request_body off;
+    proxy_set_header        Content-Length "";
+}
+
+# Redirect to Authentik when unauthenticated
+location @goauthentik_proxy_signin {
+    internal;
+    auth_request off;
+    add_header Set-Cookie $auth_cookie;
+    return 302 https://authentik.snarfnet.net/outpost.goauthentik.io/start?rd=$scheme://$http_host$request_uri;
+}
+
+# Protected Topology HTML (Interactive)
+location = /topology.html {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.html;
+    default_type text/html;
+}
+
+# Protected Topology PNG (High-Res Image)
+location = /topology.png {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.png;
+    default_type image/png;
+}
+
+location = /snarfnet_homelab_topology.html {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.html;
+    default_type text/html;
+}
+
+location = /snarfnet_homelab_topology.png {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.png;
+    default_type image/png;
+}
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/adv_config_h21.txt")
+stdin.write(adv_config)
+stdin.close()
+stdout.read()
+
+remote_script = """import sqlite3, shutil, os
+
+db_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'
+conf_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/21.conf'
+
+with open('/tmp/adv_config_h21.txt', 'r') as f:
+    adv = f.read()
+
+# Update DB
+conn = sqlite3.connect(db_path)
+cur = conn.cursor()
+cur.execute("UPDATE proxy_host SET advanced_config = ? WHERE id = 21", (adv,))
+conn.commit()
+conn.close()
+print("Updated database for host 21.")
+
+# Update 21.conf
+with open(conf_path, 'r') as f:
+    conf = f.read()
+
+# Strip existing adv_config block if present
+tag = '# Buffer tuning for Authentik headers'
+if tag in conf:
+    idx_start = conf.find(tag)
+    idx_end = conf.find('location / {', idx_start)
+    if idx_start != -1 and idx_end != -1:
+        new_conf = conf[:idx_start] + adv + "\\n\\n  " + conf[idx_end:]
+    else:
+        new_conf = conf
+else:
+    idx = conf.find('location / {')
+    if idx != -1:
+        new_conf = conf[:idx] + adv + "\\n\\n  " + conf[idx:]
+    else:
+        new_conf = conf.rstrip().rstrip('}') + "\\n\\n" + adv + "\\n}\\n"
+
+with open(conf_path, 'w') as f:
+    f.write(new_conf)
+print("Updated 21.conf.")
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/apply_h21.py")
+stdin.write(remote_script)
+stdin.close()
+stdout.read()
+
+stdin, stdout, stderr = c.exec_command("python3 /tmp/apply_h21.py")
+print("Remote python stdout:", stdout.read().decode())
+print("Remote python stderr:", stderr.read().decode())
+
+stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -t")
+out = stdout.read().decode()
+err = stderr.read().decode()
+print("nginx -t stdout:", out)
+print("nginx -t stderr:", err)
+
+if "successful" in out or "successful" in err:
+    print("Nginx syntax OK! Reloading nginx...")
+    stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
+    print("Reload stdout:", stdout.read().decode())
+    print("Reload stderr:", stderr.read().decode())
+
+c.close()
--- a/scripts/apply_homepage.py
+++ b/scripts/apply_homepage.py
@@ -0,0 +1,31 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    if out:
+        print(out)
+    if err:
+        print("[stderr]", err)
+    return out
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+stdin, stdout, stderr = c.exec_command('cat > /tmp/homepage.yaml')
+stdin.write(content)
+stdin.close()
+stdout.read()
+
+run("ls -lh /tmp/homepage.yaml")
+run("kubectl apply -f /tmp/homepage.yaml")
+run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure")
+run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s")
+
+c.close()
--- a/scripts/apply_npm_fixes.py
+++ b/scripts/apply_npm_fixes.py
@@ -0,0 +1,140 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+print("1. Creating backup of database and configs...")
+backup_cmd = """
+cp /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite.bak.$(date +%s)
+"""
+stdin, stdout, stderr = c.exec_command(backup_cmd)
+stdout.channel.recv_exit_status()
+print("Backup complete.")
+
+print("2. Updating database.sqlite for hosts 45, 41, and 37...")
+py_db = """
+import sqlite3, datetime
+conn = sqlite3.connect("/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite")
+cur = conn.cursor()
+now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+updates = [
+    (45, "140.44.4.34", 443, "https"), # Longhorn -> Aerodrome Traefik
+    (41, "140.44.4.62", 443, "https"), # Semaphore -> Spaceport Traefik
+    (37, "140.44.4.34", 443, "https"), # Uptime Kuma -> Aerodrome Traefik
+]
+
+for pid, host, port, scheme in updates:
+    cur.execute('''UPDATE proxy_host
+        SET forward_host = ?, forward_port = ?, forward_scheme = ?,
+            allow_websocket_upgrade = 1, http2_support = 1, modified_on = ?
+        WHERE id = ?''', (host, port, scheme, now, pid))
+    print(f"DB updated: Host {pid} -> {scheme}://{host}:{port}")
+
+conn.commit()
+conn.close()
+"""
+stdin, stdout, stderr = c.exec_command(f"python3 -c '{py_db}'")
+print(stdout.read().decode())
+err = stderr.read().decode()
+if err: print("DB err:", err)
+
+print("3. Generating updated nginx configuration files...")
+
+def generate_conf(pid, domains, cert_id, host, port, scheme):
+    return f"""# ------------------------------------------------------------
+# {domains}
+# ------------------------------------------------------------
+
+map $scheme $hsts_header {{
+    https   "max-age=63072000; preload";
+}}
+
+server {{
+  set $forward_scheme {scheme};
+  set $server         "{host}";
+  set $port           {port};
+
+  listen 80;
+  listen [::]:80;
+
+  listen 443 ssl;
+  listen [::]:443 ssl;
+
+  server_name {domains};
+  http2 on;
+
+  # Let's Encrypt SSL
+  include conf.d/include/letsencrypt-acme-challenge.conf;
+  include conf.d/include/ssl-cache.conf;
+  include conf.d/include/ssl-ciphers.conf;
+  ssl_certificate /etc/letsencrypt/live/npm-{cert_id}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/npm-{cert_id}/privkey.pem;
+
+  # Asset Caching
+  include conf.d/include/assets.conf;
+
+  # Block Exploits
+  include conf.d/include/block-exploits.conf;
+
+  # Force SSL
+  set $trust_forwarded_proto "F";
+  include conf.d/include/force-ssl.conf;
+
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $http_connection;
+  proxy_http_version 1.1;
+
+  access_log /data/logs/proxy-host-{pid}_access.log proxy;
+  error_log /data/logs/proxy-host-{pid}_error.log warn;
+
+  location / {{
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $http_connection;
+    proxy_http_version 1.1;
+
+    # Proxy!
+    include conf.d/include/proxy.conf;
+  }}
+
+  # Custom
+  include /data/nginx/custom/server_proxy[.]conf;
+}}
+"""
+
+configs = [
+    (45, "longhorn.aerodrome.snarfnet.net", 37, "140.44.4.34", 443, "https"),
+    (41, "semaphore.snarfnet.net", 33, "140.44.4.62", 443, "https"),
+    (37, "uptime.snarfnet.net", 27, "140.44.4.34", 443, "https"),
+]
+
+for pid, dom, cert_id, host, port, scheme in configs:
+    cfg = generate_conf(pid, dom, cert_id, host, port, scheme)
+    path = f"/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/{pid}.conf"
+    cmd = f"cat << 'EOF' > {path}\n{cfg}\nEOF\n"
+    stdin, stdout, stderr = c.exec_command(cmd)
+    stdout.channel.recv_exit_status()
+    print(f"Wrote config: {path}")
+
+print("4. Testing Nginx config inside container...")
+stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -t")
+out = stdout.read().decode()
+err = stderr.read().decode()
+print("nginx -t stdout:", out)
+print("nginx -t stderr:", err)
+
+if "successful" in out or "successful" in err:
+    print("5. Reloading Nginx via Swarm service update...")
+    stdin, stdout, stderr = c.exec_command("docker service update --force nginx-proxy-manager_app")
+    print(stdout.read().decode())
+    print("Reload completed.")
+else:
+    print("ERROR: Nginx config test failed!")
+
+c.close()
+
--- a/scripts/apply_pihole_redirect.py
+++ b/scripts/apply_pihole_redirect.py
@@ -0,0 +1,26 @@
+import paramiko
+
+def apply_manifest(host, cluster_name, file_path):
+    print(f"Applying {file_path} to {cluster_name} ({host})...")
+    with open(file_path, 'r') as f:
+        manifest_content = f.read()
+
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(host, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    stdin, stdout, stderr = ssh.exec_command('k3s kubectl apply -f -')
+    stdin.write(manifest_content)
+    stdin.channel.shutdown_write()
+
+    out = stdout.read().decode('utf-8')
+    err = stderr.read().decode('utf-8')
+    print("Output:")
+    print(out)
+    if err:
+        print("Stderr:", err)
+    ssh.close()
+
+apply_manifest('140.44.4.11', 'Aerodrome', r'homelab-gitops-repo/manifests/aerodrome/dns/dns-stack.yaml')
+apply_manifest('140.44.4.41', 'Spaceport', r'homelab-gitops-repo/manifests/spaceport/dns/dns-stack.yaml')
+
--- a/scripts/apply_prom_grafana.py
+++ b/scripts/apply_prom_grafana.py
@@ -0,0 +1,45 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r") as f:
+    prom_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml")
+stdin.write(prom_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying Prometheus:")
+print(run("kubectl apply -f /tmp/prometheus.yaml"))
+
+with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r") as f:
+    grafana_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_full.yaml")
+stdin.write(grafana_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("\nApplying Grafana configuration & dashboards:")
+print(run("kubectl apply -f /tmp/grafana_full.yaml"))
+
+print("\nRestarting Grafana:")
+print(run("kubectl rollout restart deployment/grafana -n default"))
+print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
+
+print("\nChecking Prometheus pod:")
+time.sleep(5)
+print(run("kubectl get pods -l app.kubernetes.io/name=prometheus"))
+
+c.close()
+
--- a/scripts/apply_prom_update.py
+++ b/scripts/apply_prom_update.py
@@ -0,0 +1,36 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f:
+    prom_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_updated.yaml")
+stdin.write(prom_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying Prometheus:")
+print(run("kubectl apply -f /tmp/prometheus_updated.yaml"))
+
+print("\nRestarting Prometheus:")
+print(run("kubectl rollout restart deployment/prometheus -n default"))
+print(run("kubectl rollout status deployment/prometheus -n default --timeout=60s"))
+
+print("\nWaiting 15 seconds for scrape loop...")
+time.sleep(15)
+
+print("\nPrometheus active targets count:")
+print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets | grep -o '\"health\":\"up\"' | wc -l"))
+
+c.close()
+
--- a/scripts/apply_semaphore_fix.py
+++ b/scripts/apply_semaphore_fix.py
@@ -0,0 +1,72 @@
+import paramiko
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+print("1. Updating Authentik Provider 10 redirect_uris...")
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+provider_update = {
+    "redirect_uris": [
+        {"matching_mode": "strict", "url": "https://semaphore.snarfnet.net/auth/oidc/authentik/redirect"},
+        {"matching_mode": "strict", "url": "https://semaphore.snarfnet.net/api/auth/oidc/authentik/redirect/"},
+        {"matching_mode": "regex", "url": "https://semaphore\\.snarfnet\\.net/.*"}
+    ]
+}
+
+req = urllib.request.Request(
+    "https://authentik.snarfnet.net/api/v3/providers/oauth2/10/",
+    headers=headers,
+    data=json.dumps(provider_update).encode('utf-8'),
+    method="PATCH"
+)
+try:
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        res = json.loads(resp.read().decode())
+        print("  ✓ Authentik Provider 10 updated successfully:", res.get('redirect_uris'))
+except Exception as e:
+    print("  ✗ Failed to update Authentik Provider 10:", e)
+
+print("\n2. Applying updated Semaphore manifest to Spaceport (140.44.4.41)...")
+manifest_path = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\semaphore\semaphore.yaml"
+with open(manifest_path, 'r', encoding='utf-8') as f:
+    content = f.read()
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl apply -f -")
+stdin.write(content)
+stdin.channel.shutdown_write()
+print("Apply output:", stdout.read().decode())
+err = stderr.read().decode()
+if err: print("Apply stderr:", err)
+
+print("3. Rolling out Semaphore deployment...")
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl rollout restart deployment/semaphore -n infrastructure; k3s kubectl rollout status deployment/semaphore -n infrastructure --timeout=90s")
+print(stdout.read().decode())
+
+# Check new pod env
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o jsonpath='{.items[0].metadata.name}'")
+new_pod = stdout.read().decode().strip()
+print(f"New pod: {new_pod}")
+
+stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n infrastructure {new_pod} -- env | grep SEMAPHORE_OIDC_PROVIDERS")
+print("New env:", stdout.read().decode())
+
+ssh.close()
+print("Done!")
+
--- a/scripts/apply_synology_fixes.py
+++ b/scripts/apply_synology_fixes.py
@@ -0,0 +1,161 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+print("1. Updating SQLite database for hosts 11 and 28...")
+dsm_adv = """proxy_buffer_size 128k;
+proxy_buffers 8 256k;
+proxy_busy_buffers_size 256k;
+proxy_read_timeout 3600s;
+proxy_send_timeout 3600s;
+proxy_ssl_verify off;"""
+
+py_db = f"""
+import sqlite3, datetime
+conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
+cur = conn.cursor()
+now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+adv = '''{dsm_adv}'''
+
+# Host 11: sn-nas.snarfnet.net -> 14.10.10.71:5001
+cur.execute('''UPDATE proxy_host
+    SET forward_host = "14.10.10.71", forward_port = 5001, forward_scheme = "https",
+        allow_websocket_upgrade = 1, http2_support = 1, advanced_config = ?, modified_on = ?
+    WHERE id = 11''', (adv, now))
+
+# Host 28: snarfnas.snarfnet.net -> 14.10.10.72:5001 (Clean DSM, remove dead outpost)
+cur.execute('''UPDATE proxy_host
+    SET forward_host = "14.10.10.72", forward_port = 5001, forward_scheme = "https",
+        allow_websocket_upgrade = 1, http2_support = 1, advanced_config = ?, modified_on = ?
+    WHERE id = 28''', (adv, now))
+
+conn.commit()
+print("Database updated for hosts 11 and 28.")
+conn.close()
+"""
+
+sftp = c.open_sftp()
+with sftp.open('/tmp/update_nas_db.py', 'w') as f:
+    f.write(py_db)
+sftp.close()
+
+stdin, stdout, stderr = c.exec_command("python3 /tmp/update_nas_db.py")
+print(stdout.read().decode())
+err = stderr.read().decode()
+if err: print("DB err:", err)
+
+print("2. Generating Nginx configuration files for hosts 11 and 28...")
+
+def make_dsm_conf(pid, domain, cert_id, host, port):
+    return f"""# ------------------------------------------------------------
+# {domain}
+# ------------------------------------------------------------
+
+map $scheme $hsts_header {{
+    https   "max-age=63072000;includeSubDomains; preload";
+}}
+
+server {{
+  set $forward_scheme https;
+  set $server         "{host}";
+  set $port           {port};
+
+  listen 80;
+  listen [::]:80;
+
+  listen 443 ssl;
+  listen [::]:443 ssl;
+
+  server_name {domain};
+  http2 on;
+
+  # Let's Encrypt SSL
+  include conf.d/include/letsencrypt-acme-challenge.conf;
+  include conf.d/include/ssl-cache.conf;
+  include conf.d/include/ssl-ciphers.conf;
+  ssl_certificate /etc/letsencrypt/live/npm-{cert_id}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/npm-{cert_id}/privkey.pem;
+
+  # Asset Caching
+  include conf.d/include/assets.conf;
+
+  # Block Exploits
+  include conf.d/include/block-exploits.conf;
+
+  # HSTS
+  add_header Strict-Transport-Security $hsts_header always;
+
+  # Force SSL
+  set $trust_forwarded_proto "F";
+  include conf.d/include/force-ssl.conf;
+
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $http_connection;
+  proxy_http_version 1.1;
+
+  access_log /data/logs/proxy-host-{pid}_access.log proxy;
+  error_log /data/logs/proxy-host-{pid}_error.log warn;
+
+  location / {{
+    # HSTS
+    add_header Strict-Transport-Security $hsts_header always;
+
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $http_connection;
+    proxy_http_version 1.1;
+
+    # DSM Buffer Tuning
+    proxy_buffer_size 128k;
+    proxy_buffers 8 256k;
+    proxy_busy_buffers_size 256k;
+    proxy_read_timeout 3600s;
+    proxy_send_timeout 3600s;
+    proxy_ssl_verify off;
+
+    # Proxy!
+    include conf.d/include/proxy.conf;
+  }}
+
+  # Custom
+  include /data/nginx/custom/server_proxy[.]conf;
+}}
+"""
+
+conf11 = make_dsm_conf(11, "sn-nas.snarfnet.net", 12, "14.10.10.71", 5001)
+conf28 = make_dsm_conf(28, "snarfnas.snarfnet.net", 22, "14.10.10.72", 5001)
+
+path11 = "/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/11.conf"
+path28 = "/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/28.conf"
+
+stdin, stdout, stderr = c.exec_command(f"cat << 'EOF' > {path11}\n{conf11}\nEOF\n")
+stdout.channel.recv_exit_status()
+print(f"Wrote {path11}")
+
+stdin, stdout, stderr = c.exec_command(f"cat << 'EOF' > {path28}\n{conf28}\nEOF\n")
+stdout.channel.recv_exit_status()
+print(f"Wrote {path28}")
+
+print("3. Testing Nginx configuration...")
+stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -t")
+out = stdout.read().decode()
+err = stderr.read().decode()
+print("nginx -t stdout:", out)
+print("nginx -t stderr:", err)
+
+if "successful" in out or "successful" in err:
+    print("4. Reloading Nginx Swarm service...")
+    stdin, stdout, stderr = c.exec_command("docker service update --force nginx-proxy-manager_app")
+    print(stdout.read().decode())
+    print("Reload complete!")
+else:
+    print("ERROR: Nginx config invalid!")
+
+c.close()
+
--- a/scripts/apply_web_vips.py
+++ b/scripts/apply_web_vips.py
@@ -0,0 +1,22 @@
+import paramiko
+import sys
+import base64
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+def apply_manifest(host, file_path):
+    with open(file_path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode('ascii')
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(host, username="root", key_filename=key_path, timeout=5)
+    stdin, stdout, stderr = c.exec_command(f"echo '{b64}' | base64 -d | k3s kubectl apply -f -")
+    print(f"Apply on {host}:")
+    print(stdout.read().decode('utf-8'))
+    c.close()
+
+apply_manifest("140.44.4.11", r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml")
+apply_manifest("140.44.4.41", r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml")
+
--- a/scripts/audit_and_pin_vips.py
+++ b/scripts/audit_and_pin_vips.py
@@ -0,0 +1,53 @@
+import paramiko
+import json
+
+TARGET_VIPS = {
+    ("monitoring", "uptime-lb"): "140.44.4.28",
+    ("infra", "authentik-lb"): "140.44.4.29",
+    ("awx", "awx-lb"): "140.44.4.30",
+    ("infra", "technitium-npm-vip-service"): "140.44.4.31",
+    ("default", "grafana-lb"): "140.44.4.32",
+    ("infra", "unified-npm-vip-service"): "140.44.4.33",
+    ("kube-system", "traefik"): "140.44.4.34",
+    ("infra", "ak-outpost-authentik-ldap-outpost"): "140.44.4.35",
+    ("longhorn-system", "longhorn-frontend-lb"): "140.44.4.36",
+}
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+    print("=== PINNING AERODROME LOADBALANCER VIPS ===")
+    for (ns, name), target_ip in TARGET_VIPS.items():
+        patch = json.dumps({
+            "metadata": {
+                "annotations": {
+                    "metallb.universe.tf/loadBalancerIP": target_ip
+                }
+            },
+            "spec": {
+                "loadBalancerIP": target_ip
+            }
+        })
+        
+        cmd = f"kubectl patch svc {name} -n {ns} --type=merge -p '{patch}'"
+        stdin, stdout, stderr = ssh.exec_command(cmd)
+        out = stdout.read().decode().strip()
+        err = stderr.read().decode().strip()
+        print(f"[{ns}/{name}] -> {target_ip}: {out} {err}")
+
+    print("\n=== VERIFYING FINAL ALLOCATION STATE ===")
+    for (ns, name), target_ip in TARGET_VIPS.items():
+        stdin, stdout, stderr = ssh.exec_command(f"kubectl get svc {name} -n {ns} -o json")
+        svc = json.loads(stdout.read().decode())
+        current_ingress = [x.get('ip') for x in svc.get('status', {}).get('loadBalancer', {}).get('ingress', [])]
+        spec_ip = svc.get('spec', {}).get('loadBalancerIP')
+        ann_ip = svc.get('metadata', {}).get('annotations', {}).get('metallb.universe.tf/loadBalancerIP')
+        status = "LOCKED" if (spec_ip == target_ip and ann_ip == target_ip and target_ip in current_ingress) else "MISMATCH"
+        print(f"[{status}] {ns}/{name}: Ingress={current_ingress}, Spec={spec_ip}, Annotation={ann_ip}")
+
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
--- a/scripts/audit_authentik.py
+++ b/scripts/audit_authentik.py
@@ -0,0 +1,30 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+def run_query(title, q):
+    cmd = f"""k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"{q}\\"" """
+    stdin, stdout, stderr = c.exec_command(cmd)
+    print(f"=== {title} ===")
+    out = stdout.read().decode('utf-8')
+    err = stderr.read().decode('utf-8')
+    if out.strip(): print(out.strip())
+    if err.strip(): print("ERR:", err.strip())
+    print()
+
+run_query("OUTPOST PROVIDERS", "SELECT outpost_id, provider_id FROM authentik_outposts_outpost_providers;")
+
+
+
+
+
+
+
+
+c.close()
--- a/scripts/audit_cluster_workloads.py
+++ b/scripts/audit_cluster_workloads.py
@@ -0,0 +1,21 @@
+import paramiko
+
+def inspect_cluster(ip, name):
+    print(f"\n=======================================================")
+    print(f"=== {name} WORKLOADS ({ip}) ===")
+    print(f"=======================================================")
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    
+    cmd = "kubectl get deploy,sts,ds -A -o custom-columns=NAMESPACE:.metadata.namespace,KIND:.kind,NAME:.metadata.name,IMAGES:.spec.template.spec.containers[*].image"
+    _, out, _ = ssh.exec_command(cmd)
+    lines = out.read().decode().splitlines()
+    for l in lines:
+        if not any(k in l for k in ['kube-system', 'cattle-', 'metallb-system', 'longhorn-system']):
+            print(l)
+    ssh.close()
+
+inspect_cluster('140.44.4.11', 'AERODROME')
+inspect_cluster('140.44.4.41', 'SPACEPORT')
+
--- a/scripts/audit_gitops_manifests.py
+++ b/scripts/audit_gitops_manifests.py
@@ -0,0 +1,40 @@
+import os
+import re
+import paramiko
+
+manifest_root = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests"
+
+print("=== GITOPS MANIFEST IMAGE AUDIT ===")
+
+image_re = re.compile(r'image:\s*([^\s"\']+)', re.IGNORECASE)
+chart_re = re.compile(r'chart:\s*([^\s"\']+)', re.IGNORECASE)
+version_re = re.compile(r'version:\s*([^\s"\']+)', re.IGNORECASE)
+
+manifest_data = {}
+
+for root, dirs, files in os.walk(manifest_root):
+    for f in files:
+        if f.endswith('.yaml') or f.endswith('.yml'):
+            path = os.path.join(root, f)
+            rel_path = os.path.relpath(path, manifest_root)
+            with open(path, 'r', encoding='utf-8', errors='ignore') as fp:
+                content = fp.read()
+                images = image_re.findall(content)
+                charts = chart_re.findall(content)
+                versions = version_re.findall(content)
+                if images or charts:
+                    manifest_data[rel_path] = {
+                        'images': list(set(images)),
+                        'charts': list(set(charts)),
+                        'versions': list(set(versions))
+                    }
+
+for rel_path, data in sorted(manifest_data.items()):
+    print(f"\nFile: {rel_path}")
+    if data['images']:
+        print("  Images:", ", ".join(data['images']))
+    if data['charts']:
+        print("  Charts:", ", ".join(data['charts']))
+    if data['versions']:
+        print("  Versions:", ", ".join(data['versions']))
+
--- a/scripts/audit_harvester_hardware.py
+++ b/scripts/audit_harvester_hardware.py
@@ -0,0 +1,75 @@
+import paramiko
+import time
+import re
+
+nodes = {
+    "Node 1 (snaihvs0lnx01)": "140.44.44.121",
+    "Node 2 (snaihvs0lnx02)": "140.44.44.122",
+    "Node 3 (snaihvs0lnx03)": "140.44.44.123",
+    "Node 4 (idrac-spacedock4)": "140.44.44.124",
+}
+
+results = {}
+
+for name, ip in nodes.items():
+    print(f"Connecting to {name} ({ip})...")
+    node_data = {}
+    try:
+        c = paramiko.SSHClient()
+        c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        c.connect(ip, username="root", password="Katzir476!", timeout=10)
+        chan = c.invoke_shell()
+        time.sleep(2)
+        if chan.recv_ready():
+            chan.recv(4096)
+            
+        def send_and_recv(cmd, wait_time=2.5):
+            chan.send(cmd + "\r\n")
+            time.sleep(wait_time)
+            out = ""
+            while chan.recv_ready():
+                out += chan.recv(8192).decode('utf-8', errors='ignore')
+                time.sleep(0.5)
+            return out
+
+        # 1. BIOS Settings
+        raw_bios = send_and_recv("racadm get bios.SysProfileSettings", 2)
+        for key in ["SysProfile", "ProcCStates", "ProcC1E", "ProcTurboMode", "ProcPwrPerf"]:
+            m = re.search(rf"{key}\s*=\s*(\w+)", raw_bios)
+            node_data[key] = m.group(1) if m else "Unknown"
+
+        raw_proc = send_and_recv("racadm get bios.ProcSettings", 2)
+        for key in ["ProcVirtualization", "ProcExecuteDisable", "LogicalProc"]:
+            m = re.search(rf"{key}\s*=\s*(\w+)", raw_proc)
+            node_data[key] = m.group(1) if m else "Unknown"
+
+        raw_dev = send_and_recv("racadm get bios.IntegratedDevices", 2)
+        for key in ["SriovGlobalEnable"]:
+            m = re.search(rf"{key}\s*=\s*(\w+)", raw_dev)
+            node_data[key] = m.group(1) if m else "Unknown"
+
+        # 2. Storage / RAID
+        raw_ctrl = send_and_recv("racadm raid get controllers", 2)
+        controllers = [line.strip() for line in raw_ctrl.splitlines() if "RAID." in line or "PERC" in line or "AHCI." in line]
+        node_data["Controllers"] = ", ".join(controllers) if controllers else "None"
+
+        raw_vdisks = send_and_recv("racadm raid get vdisks", 3)
+        vdisks = [line.strip() for line in raw_vdisks.splitlines() if line.startswith("Disk.Virtual")]
+        node_data["VirtualDisks"] = len(vdisks)
+        node_data["VirtualDisksList"] = vdisks
+
+        raw_pdisks = send_and_recv("racadm raid get pdisks", 4)
+        pdisk_lines = [line.strip() for line in raw_pdisks.splitlines() if line.startswith("Disk.Bay")]
+        node_data["PhysicalDisks"] = len(pdisk_lines)
+        node_data["PDisksSummary"] = raw_pdisks
+
+        c.close()
+    except Exception as e:
+        node_data["Error"] = str(e)
+
+    results[name] = node_data
+
+print("\n\n========================= SUMMARY MATRIX =========================")
+import json
+print(json.dumps(results, indent=2))
+
--- a/scripts/audit_harvester_live_nodes.py
+++ b/scripts/audit_harvester_live_nodes.py
@@ -0,0 +1,24 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', password='Katzir476!', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
+    return stdout.read().decode().strip()
+
+print("=== CPU & VIRTUALIZATION (lscpu) ===")
+print(run("lscpu | grep -E 'Model name|CPU\(s\):|Thread|Socket|Virtualization|NUMA'"))
+
+print("\n=== KERNEL PARAMETERS (/proc/cmdline) ===")
+print(run("cat /proc/cmdline"))
+
+print("\n=== STORAGE CONTROLLER (lspci) ===")
+print(run("lspci | grep -E 'RAID|SAS|SATA|Ethernet|Network'"))
+
+print("\n=== BLOCK DEVICES (lsblk) ===")
+print(run("lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS"))
+
+c.close()
+
--- a/scripts/audit_spaceport_kernels.py
+++ b/scripts/audit_spaceport_kernels.py
@@ -0,0 +1,39 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+spaceport_nodes = [
+    ('sninfr0lnxctl01', '140.44.4.41'),
+    ('sninfr0lnxctl02', '140.44.4.42'),
+    ('sninfr0lnxctl03', '140.44.4.43'),
+    ('sninfr0lnxctl04', '140.44.4.44'),
+    ('sninfr0lnxwrk01', '140.44.4.51'),
+    ('sninfr0lnxwrk02', '140.44.4.52'),
+    ('sninfr0lnxwrk03', '140.44.4.53'),
+    ('sninfr0lnxwrk04', '140.44.4.54'),
+    ('sninfr0lnxwrk05', '140.44.4.55'),
+    ('sninfr0lnxwrk06', '140.44.4.56'),
+]
+
+print(f"{'NODE':<18} {'IP':<15} {'RUNNING KERNEL':<22} {'LATEST MODULES':<25} {'VETH LOADED':<12} {'UPTIME'}")
+print("="*105)
+
+for name, ip in spaceport_nodes:
+    try:
+        client.connect(ip, username="root", key_filename=key_path, timeout=5)
+        stdin, stdout, stderr = client.exec_command("uname -r; ls -1 /lib/modules | sort -V | tail -n 1; lsmod | grep -q veth && echo YES || echo NO; uptime -p")
+        lines = [l.strip() for l in stdout.read().decode('utf-8').strip().split('\n') if l.strip()]
+        running_k = lines[0] if len(lines) > 0 else "?"
+        latest_mod = lines[1] if len(lines) > 1 else "?"
+        veth = lines[2] if len(lines) > 2 else "?"
+        uptime_str = lines[3] if len(lines) > 3 else "?"
+        print(f"{name:<18} {ip:<15} {running_k:<22} {latest_mod:<25} {veth:<12} {uptime_str}")
+        client.close()
+    except Exception as e:
+        print(f"{name:<18} {ip:<15} ERROR: {e}")
+
--- a/scripts/audit_zabbix_alerts.py
+++ b/scripts/audit_zabbix_alerts.py
@@ -0,0 +1,48 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
+
+def zbx_req(method, params, token=None):
+    payload = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1}
+    headers = {'Content-Type': 'application/json-rpc'}
+    if token:
+        headers['Authorization'] = f'Bearer {token}'
+    req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method='POST')
+    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+        return json.loads(r.read().decode())['result']
+
+token = zbx_req('user.login', {'username': 'Admin', 'password': 'zabbix'})
+print("Zabbix Auth Token:", token)
+
+# 1. Actions
+actions = zbx_req('action.get', {'output': 'extend', 'selectOperations': 'extend', 'selectFilter': 'extend'}, token)
+print("\n=== TRIGGER ACTIONS ===")
+for a in actions:
+    print(f"ID: {a['actionid']}, Name: {a['name']}, Status (0=enabled, 1=disabled): {a['status']}, EventSource: {a['eventsource']}")
+    print(f"  Operations: {a.get('operations')}")
+    print(f"  Filter: {a.get('filter')}")
+
+# 2. Media Types
+media_types = zbx_req('mediatype.get', {'output': 'extend'}, token)
+print("\n=== MEDIA TYPES ===")
+for m in media_types:
+    print(f"ID: {m['mediatypeid']}, Name: {m['name']}, Type: {m['type']}, Status: {m['status']}")
+
+# 3. User Media
+users = zbx_req('user.get', {'output': ['userid', 'username'], 'selectMedias': 'extend'}, token)
+print("\n=== USERS & MEDIA ===")
+for u in users:
+    print(f"User: {u['username']} (ID: {u['userid']})")
+    for med in u.get('medias', []):
+        print(f"  Media: mediatypeid={med['mediatypeid']}, sendto={med['sendto']}, active={med['active']}, severity={med['severity']}")
+
+# 4. Recent Alerts
+alerts = zbx_req('alert.get', {'output': 'extend', 'limit': 15, 'sortfield': 'clock', 'sortorder': 'DESC'}, token)
+print("\n=== RECENT ALERTS ===")
+for al in alerts:
+    print(f"Alert ID: {al['alertid']}, ActionID: {al['actionid']}, Clock: {al['clock']}, Error: {al.get('error')}, Status: {al.get('status')}, Subject: {al.get('subject')}")
+
--- a/scripts/authorize_and_test.py
+++ b/scripts/authorize_and_test.py
@@ -0,0 +1,35 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+pub_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6ycMke4AaV2qzWfULZCTZOKhSKf4pY2qRGYjYzs1MGKutJxt2jkYNK2aS0XffxLRR49v23zHbBwLYraNl2Mkb9rsKsyvKn8CAa1CEl9NxOci4LDRT8CeWHRnMZn8Mdu1pfeRwM26/Bjnd4RJrSKC7fyM0OlOpEp0DgnuPBqBtPMFGw+TCUT8eZ0pBrAI94wEan4l35kmuBcqm6RH3KuD1dDJKnPLzJ3/T21CViYL0GgFIDDor/hGI45VfOI/jKmkgM4YoudTCde6uPV5e2yaJpORZlR4nDxg9ntXp8HxbHGZkXYr55Ibsjem0lMU6nBh33jGbJVvat1Ay4oNCbygJsHfC/8ppe0bnLhnasxVAXhTvklZZL/sf7K/VS0ZifpfqB6DKtr1lziOm2wQeahv+Y1uoHkVNa3XY8Fx7xGPeT1BGd9lfcbUoweoTdMPMIPaSZlNGY32AHYkM4PShG9yI1coAqXzMby0+Z/Qwtr4l6cLKVvcGgqh5sd+z1+YvZbs= root@SNINFR2LNXCTL01"
+
+# 1. Authorize on 140.44.4.71
+c71 = paramiko.SSHClient()
+c71.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c71.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+cmd = f"""
+mkdir -p ~/.ssh && chmod 700 ~/.ssh
+grep -qF '{pub_key}' ~/.ssh/authorized_keys 2>/dev/null || echo '{pub_key}' >> ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+"""
+c71.exec_command(cmd)
+c71.close()
+print("Authorized 140.44.4.11 key on 140.44.4.71.")
+
+# 2. Test SSH from 140.44.4.11 to 140.44.4.71
+c11 = paramiko.SSHClient()
+c11.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c11.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = c11.exec_command("ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.71 hostname")
+out = stdout.read().decode('utf-8').strip()
+err = stderr.read().decode('utf-8').strip()
+print("Result of ssh from 140.44.4.11 to 140.44.4.71:")
+print("STDOUT:", out)
+if err: print("STDERR:", err)
+c11.close()
+
--- a/scripts/capture_screenshot.py
+++ b/scripts/capture_screenshot.py
@@ -0,0 +1,27 @@
+import subprocess
+import os
+
+edge_path = r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe"
+out_png = os.path.abspath("docs/snarfnet_homelab_topology.png")
+html_url = "file:///" + os.path.abspath("docs/snarfnet_homelab_topology.html").replace("\\", "/")
+
+cmd = [
+    edge_path,
+    "--headless",
+    "--disable-gpu",
+    "--no-sandbox",
+    "--hide-scrollbars",
+    "--window-size=1920,2400",
+    f"--screenshot={out_png}",
+    html_url
+]
+
+print("Running cmd:", " ".join(cmd))
+res = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+print("Return code:", res.returncode)
+print("Stdout:", res.stdout)
+print("Stderr:", res.stderr)
+print("File exists?", os.path.exists(out_png))
+if os.path.exists(out_png):
+    print("File size:", os.path.getsize(out_png))
+
--- a/scripts/cat_21_conf.py
+++ b/scripts/cat_21_conf.py
@@ -0,0 +1,10 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+cmd = "docker exec $(docker ps -q --filter name=nginx-proxy-manager_app) cat /data/nginx/proxy_host/21.conf"
+_, o, _ = c.exec_command(cmd)
+print(o.read().decode())
+c.close()
+
--- a/scripts/cat_prom_conf.py
+++ b/scripts/cat_prom_conf.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl exec deployment/prometheus -- cat /etc/prometheus/prometheus.yml"))
+c.close()
+
--- a/scripts/check_13_conf.py
+++ b/scripts/check_13_conf.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /data/nginx/proxy_host/13.conf | grep server_name')
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_214_pods.py
+++ b/scripts/check_214_pods.py
@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -o wide")
+print("=== PODS IN CATTLE-SYSTEM ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
+print("=== ROLLOUT STATUS ===")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
+print("=== ENDPOINT ===")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+
--- a/scripts/check_215_pods.py
+++ b/scripts/check_215_pods.py
@@ -0,0 +1,29 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -o wide")
+print("=== PODS IN CATTLE-SYSTEM ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
+print("=== ROLLOUT STATUS ===")
+print(stdout.read().decode('utf-8'))
+print(stderr.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
+print("=== ENDPOINT ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -n cattle-system")
+print("=== HELM LIST ===")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+
--- a/scripts/check_33_conf.py
+++ b/scripts/check_33_conf.py
@@ -0,0 +1,12 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.71", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/33.conf'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("33.conf:\n", stdout.read().decode())
+
+client.close()
+
--- a/scripts/check_43_access.py
+++ b/scripts/check_43_access.py
@@ -0,0 +1,13 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-43_access.log")
+print("=== Access log 43 (pihole2) around 11:00 ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_49_conf.py
+++ b/scripts/check_49_conf.py
@@ -0,0 +1,12 @@
+import paramiko
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.71", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
+
+cmd = 'cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/49.conf'
+stdin, stdout, stderr = client.exec_command(cmd)
+print("49.conf (ai.snarfnet.net):\n", stdout.read().decode())
+
+client.close()
+
--- a/scripts/check_51.py
+++ b/scripts/check_51.py
@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("ss -tulpn | grep ':80 '")
+run("ip a")
+run("kubectl get pod -A -o wide | grep sninfr4lnxwrk01 | grep pihole || true")
+ssh.close()
+
--- a/scripts/check_51_nat.py
+++ b/scripts/check_51_nat.py
@@ -0,0 +1,18 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("iptables -t nat -S | grep -E '80\b|31905|31937'")
+run("crictl ps")
+ssh.close()
+
--- a/scripts/check_55_and_43.py
+++ b/scripts/check_55_and_43.py
@@ -0,0 +1,17 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/55.conf")
+print("=== 55.conf (pihole1) ===")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/43.conf")
+print("=== 43.conf (pihole2) ===")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_71_containers.py
+++ b/scripts/check_71_containers.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_aero_health.py
+++ b/scripts/check_aero_health.py
@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+
+stdin, stdout, stderr = client.exec_command('k3s kubectl get pods -n cattle-system')
+print("=== AERODROME CATTLE-SYSTEM PODS ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_aero_infra.py
+++ b/scripts/check_aero_infra.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get deploy,svc,pod")
+print(stdout.read().decode('utf-8'))
+c.close()
+
--- a/scripts/check_aero_metrics_sources.py
+++ b/scripts/check_aero_metrics_sources.py
@@ -0,0 +1,24 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Nodes on Aerodrome:")
+print(run("kubectl get nodes -o wide"))
+
+print("\nDaemonSets on Aerodrome:")
+print(run("kubectl get ds -A"))
+
+print("\nServices matching metrics:")
+print(run("kubectl get svc -A | grep -E 'metric|node|cadvisor|exporter'"))
+
+print("\nKubelet metrics on node 140.44.4.11 (port 10250 or 10255):")
+print(run("curl -sk https://localhost:10250/metrics/cadvisor | head -n 10"))
+
+c.close()
+
--- a/scripts/check_aero_pihole_auth.py
+++ b/scripts/check_aero_pihole_auth.py
@@ -0,0 +1,12 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/pihole -- curl -s -X POST "http://localhost/api/auth" -H "Content-Type: application/json" -d \'{"password":"snarfnetadmin"}\'')
+print("Aerodrome API auth test:")
+print(stdout.read().decode('utf-8', errors='ignore'))
+
+ssh.close()
+
--- a/scripts/check_aero_svc.py
+++ b/scripts/check_aero_svc.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get svc")
+print(stdout.read().decode('utf-8'))
+c.close()
+
--- a/scripts/check_aerodrome_pods.py
+++ b/scripts/check_aerodrome_pods.py
@@ -0,0 +1,15 @@
+import paramiko
+
+s = paramiko.SSHClient()
+s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    _, out, _ = s.exec_command(cmd)
+    print(out.read().decode())
+
+run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
+run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
+s.close()
+
--- a/scripts/check_aerodrome_preflight.py
+++ b/scripts/check_aerodrome_preflight.py
@@ -0,0 +1,19 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("kubectl get nodes")
+run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
+run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness | grep -v healthy || true")
+ssh.close()
+
--- a/scripts/check_aerodrome_quick.py
+++ b/scripts/check_aerodrome_quick.py
@@ -0,0 +1,8 @@
+import paramiko
+ssh=paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+_,o,_=ssh.exec_command('kubectl get nodes')
+print(o.read().decode())
+ssh.close()
+
--- a/scripts/check_aerodrome_vip_details.py
+++ b/scripts/check_aerodrome_vip_details.py
@@ -0,0 +1,29 @@
+import paramiko
+import json
+
+def main():
+    s = paramiko.SSHClient()
+    s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet')
+
+    _, o, _ = s.exec_command('kubectl get svc -A -o json')
+    data = json.loads(o.read().decode())
+    print("=== AERODROME LOADBALANCER SERVICES ===")
+    for item in data.get('items', []):
+        if item.get('spec', {}).get('type') == 'LoadBalancer':
+            name = item['metadata']['name']
+            ns = item['metadata']['namespace']
+            lb_ip = item.get('spec', {}).get('loadBalancerIP')
+            ing = [x.get('ip') for x in item.get('status', {}).get('loadBalancer', {}).get('ingress', [])]
+            annotations = item['metadata'].get('annotations', {})
+            print(f"- {ns}/{name}: status_ingress={ing}, spec_loadBalancerIP={lb_ip}, metallb_annotations={list(annotations.keys())}")
+
+    _, o, _ = s.exec_command('kubectl get ipaddresspools.metallb.io -A -o yaml 2>/dev/null || kubectl get cm -n metallb-system config -o yaml 2>/dev/null || true')
+    print("\n=== METALLB CONFIG ===")
+    print(o.read().decode())
+
+    s.close()
+
+if __name__ == '__main__':
+    main()
+
--- a/scripts/check_after_reboot.py
+++ b/scripts/check_after_reboot.py
@@ -0,0 +1,29 @@
+import paramiko
+import sys
+import time
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+print("=== CHECKING VETH ON SNINFR0LNXCTL03 ===")
+client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("modprobe veth; lsmod | grep veth; systemctl is-active k3s-server || systemctl is-active k3s")
+print(stdout.read().decode('utf-8'))
+client.close()
+
+time.sleep(5)
+
+print("=== CHECKING PODS IN CATTLE-SYSTEM ON SPACEPORT ===")
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide")
+print(stdout.read().decode('utf-8'))
+
+print("=== CHECKING POD EVENTS ===")
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system describe pod -l app=cattle-cluster-agent | grep -A 10 Events:")
+print(stdout.read().decode('utf-8'))
+
+client.close()
+
--- a/scripts/check_agent_deploy.py
+++ b/scripts/check_agent_deploy.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get deployment cattle-cluster-agent -o yaml")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_agent_logs.py
+++ b/scripts/check_agent_logs.py
@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent --tail=100")
+print("=== SPACEPORT CATTLE-CLUSTER-AGENT LOGS ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_agent_logs_tail.py
+++ b/scripts/check_agent_logs_tail.py
@@ -0,0 +1,15 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=20")
+print("=== AGENT LOGS ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_agent_now.py
+++ b/scripts/check_agent_now.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=25")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_alerts_status.py
+++ b/scripts/check_alerts_status.py
@@ -0,0 +1,19 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
+
+payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
+token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+payload = {'jsonrpc': '2.0', 'method': 'alert.get', 'params': {'output': 'extend', 'limit': 20, 'sortfield': 'clock', 'sortorder': 'DESC'}, 'id': 2}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
+alerts = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+for a in alerts:
+    print(f"ID: {a['alertid']} | Status: {a['status']} | User: {a['userid']} | Error: '{a.get('error')}' | Subject: {a.get('subject')}")
+
--- a/scripts/check_all_coder_snapshots.py
+++ b/scripts/check_all_coder_snapshots.py
@@ -0,0 +1,24 @@
+import paramiko
+
+nodes = [
+    ("wrk01", "140.44.4.51"),
+    ("wrk02", "140.44.4.52"),
+    ("wrk03", "140.44.4.53"),
+    ("wrk04", "140.44.4.54"),
+    ("wrk05", "140.44.4.55"),
+    ("wrk06", "140.44.4.56"),
+]
+
+for name, ip in nodes:
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=4)
+        _, out, _ = ssh.exec_command("find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path '*/home/coder/.config/coderv2' 2>/dev/null")
+        res = out.read().decode().strip()
+        if res:
+            print(f"[{name} - {ip}] Found:\n{res}")
+        ssh.close()
+    except Exception as e:
+        print(f"[{name}] err: {e}")
+
--- a/scripts/check_all_endpoints.py
+++ b/scripts/check_all_endpoints.py
@@ -0,0 +1,19 @@
+import subprocess
+
+endpoints = [
+    ("Argo CD", "https://argo.snarfnet.net"),
+    ("Rancher Manager", "https://aerodrome.snarfnet.net"),
+    ("AWX Console", "https://awx.snarfnet.net"),
+    ("Authentik SSO", "https://authentik.snarfnet.net"),
+    ("Uptime Kuma", "https://uptime.snarfnet.net/dashboard"),
+    ("Gitea GitOps", "http://140.44.4.51:90"),
+    ("Zabbix Server", "http://140.44.4.51:8888"),
+    ("Portainer Server", "https://140.44.4.71:9443")
+]
+
+print("=== ENDPOINT HEALTH CHECK ===")
+for name, url in endpoints:
+    res = subprocess.run(f"curl.exe -k -s -o nul -w '%{{http_code}}' {url}", capture_output=True, text=True, shell=True)
+    code = res.stdout.strip()
+    print(f"[{code}] {name} ({url})")
+
--- a/scripts/check_all_npm_hosts.py
+++ b/scripts/check_all_npm_hosts.py
@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
+
+py_query = """
+import sqlite3
+conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
+cur = conn.cursor()
+cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host')
+for r in cur.fetchall():
+    print(r)
+conn.close()
+"""
+
+stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"')
+print("All NPM hosts:\n", stdout.read().decode())
+
+c.close()
+
--- a/scripts/check_all_providers.py
+++ b/scripts/check_all_providers.py
@@ -0,0 +1,30 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+def api_call(url):
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
+        return json.loads(resp.read().decode('utf-8'))
+
+apps = api_call("https://authentik.snarfnet.net/api/v3/core/applications/?page_size=100")
+for a in apps.get('results', []):
+    print(f"App: {a['name']:30} | Slug: {a['slug']:15} | Provider: {a.get('provider')}")
+
+providers = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/?page_size=100")
+for p in providers.get('results', []):
+    print(f"Proxy Provider: {p['name']:30} | PK: {p['pk']:4} | Mode: {p['mode']:15} | Cookie Domain: {p.get('cookie_domain')}")
+
--- a/scripts/check_all_tls.py
+++ b/scripts/check_all_tls.py
@@ -0,0 +1,27 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+# Check certificates in Aerodrome across all namespaces
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("""
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secrets -A --field-selector=type=kubernetes.io/tls
+""")
+print("=== TLS SECRETS ON AERODROME ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+
+# Check certificates on Spaceport across all namespaces
+client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("""
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secrets -A --field-selector=type=kubernetes.io/tls
+""")
+print("=== TLS SECRETS ON SPACEPORT ===")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_api_status.py
+++ b/scripts/check_api_status.py
@@ -0,0 +1,12 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/technitium-dns -- curl -s "http://localhost:5380/api/status"')
+print("api/status response:")
+print(stdout.read().decode('utf-8', errors='ignore'))
+
+ssh.close()
+
--- a/scripts/check_apis.py
+++ b/scripts/check_apis.py
@@ -0,0 +1,25 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+
+client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
+
+test_cmd = """
+cd /tmp/rancher-test
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm template rancher ./rancher --namespace cattle-system --set hostname=aerodrome.snarfnet.net --set ingress.tls.source=secret | grep "apiVersion:" | sort | uniq -c
+"""
+
+stdin, stdout, stderr = client.exec_command(test_cmd)
+out = stdout.read().decode('utf-8')
+err = stderr.read().decode('utf-8')
+print("APIs used:")
+print(out)
+if err:
+    print("STDERR:", err)
+
+client.close()
--- a/scripts/check_apiurl.py
+++ b/scripts/check_apiurl.py
@@ -0,0 +1,12 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/pihole -- curl -s "http://localhost/admin/login" | grep -i "data-apiurl"')
+print("data-apiurl:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_app_dll.py
+++ b/scripts/check_app_dll.py
@@ -0,0 +1,17 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/technitium-dns -- ls -la /app')
+print("/app contents:")
+print(stdout.read().decode('utf-8', errors='ignore'))
+
+# Search for AddAuthentication in dlls
+stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/technitium-dns -- strings /app/DnsServerApp.dll /app/DnsServerCore.dll 2>/dev/null | grep -i -C 3 "AddAuthentication"')
+print("AddAuthentication strings:")
+print(stdout.read().decode('utf-8', errors='ignore'))
+
+ssh.close()
+
--- a/scripts/check_argo_apps.py
+++ b/scripts/check_argo_apps.py
@@ -0,0 +1,14 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("kubectl get applications -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
+print(stdout.read().decode())
+err = stderr.read().decode()
+if err:
+    print("ERR:", err)
+
+ssh.close()
+
--- a/scripts/check_argo_ing.py
+++ b/scripts/check_argo_ing.py
@@ -0,0 +1,22 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("ERR:", err)
+
+run("kubectl get ingress argo-ing -n infrastructure -o yaml")
+run("kubectl get svc pihole-web -n infrastructure -o yaml")
+run("kubectl get svc -n infrastructure")
+run("kubectl get svc -A | grep -E 'LoadBalancer|140.44.4.62'")
+run("curl -k -I -H 'Host: argo.snarfnet.net' http://140.44.4.62")
+run("curl -k -I -H 'Host: argo.snarfnet.net' https://140.44.4.62")
+ssh.close()
+
--- a/scripts/check_argo_secret.py
+++ b/scripts/check_argo_secret.py
@@ -0,0 +1,16 @@
+import paramiko
+import base64
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Argo initial secret:")
+print(run("kubectl get secret argocd-initial-admin-secret -n infrastructure -o jsonpath='{.data.password}'"))
+
+c.close()
+
--- a/scripts/check_argocd_apps.py
+++ b/scripts/check_argocd_apps.py
@@ -0,0 +1,73 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+try:
+    resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
+    data = resp.json()
+    items = data.get("items", [])
+    print(f"Total Applications: {len(items)}\n")
+    
+    out_of_sync = []
+    synced = []
+    
+    for app in items:
+        name = app.get("metadata", {}).get("name")
+        sync_status = app.get("status", {}).get("sync", {}).get("status")
+        health_status = app.get("status", {}).get("health", {}).get("status")
+        dest_server = app.get("spec", {}).get("destination", {}).get("server")
+        dest_ns = app.get("spec", {}).get("destination", {}).get("namespace")
+        repo = app.get("spec", {}).get("source", {}).get("repoURL")
+        path = app.get("spec", {}).get("source", {}).get("path")
+        
+        # Check operation or sync conditions/messages
+        operation_state = app.get("status", {}).get("operationState", {})
+        op_phase = operation_state.get("phase")
+        op_msg = operation_state.get("message")
+        conditions = app.get("status", {}).get("conditions", [])
+        
+        info = {
+            "name": name,
+            "sync": sync_status,
+            "health": health_status,
+            "dest_ns": dest_ns,
+            "dest_server": dest_server,
+            "repo": repo,
+            "path": path,
+            "op_phase": op_phase,
+            "op_msg": op_msg,
+            "conditions": conditions
+        }
+        
+        if sync_status != "Synced" or health_status not in ["Healthy", None]:
+            out_of_sync.append(info)
+        else:
+            synced.append(info)
+            
+    print(f"=== OUT OF SYNC / DEGRADED APPS ({len(out_of_sync)}) ===")
+    for a in out_of_sync:
+        print(f"\nApp: {a['name']}")
+        print(f"  Sync: {a['sync']} | Health: {a['health']}")
+        print(f"  Dest: {a['dest_ns']} on {a['dest_server']}")
+        print(f"  Path: {a['path']}")
+        if a['op_phase']:
+            print(f"  Op Phase: {a['op_phase']}")
+        if a['op_msg']:
+            print(f"  Op Msg: {a['op_msg']}")
+        if a['conditions']:
+            print(f"  Conditions: {a['conditions']}")
+            
+    print(f"\n=== SYNCED APPS ({len(synced)}) ===")
+    for a in synced:
+        print(f"  {a['name']}: Sync={a['sync']}, Health={a['health']}, NS={a['dest_ns']}")
+
+except Exception as e:
+    print("Error querying Argo CD API:", e)
+
--- a/scripts/check_argocd_now.py
+++ b/scripts/check_argocd_now.py
@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, out, _ = ssh.exec_command("kubectl get applications -n infrastructure -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
+print(out.read().decode())
+
+ssh.close()
+
--- a/scripts/check_argocd_resources.py
+++ b/scripts/check_argocd_resources.py
@@ -0,0 +1,29 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
+data = resp.json()
+items = data.get("items", [])
+
+for app in items:
+    name = app.get("metadata", {}).get("name")
+    sync = app.get("status", {}).get("sync", {}).get("status")
+    health = app.get("status", {}).get("health", {}).get("status")
+    resources = app.get("status", {}).get("resources", [])
+    
+    out_resources = [r for r in resources if r.get("status") != "Synced" or r.get("health", {}).get("status") not in ["Healthy", None]]
+    
+    print(f"[{name}] Sync: {sync}, Health: {health}, Total Resources: {len(resources)}")
+    if out_resources:
+        print(f"  -> Out-of-sync / unhealthy resources:")
+        for r in out_resources:
+            print(f"     {r.get('kind')}/{r.get('name')}: Sync={r.get('status')}, Health={r.get('health', {}).get('status')}")
+
--- a/scripts/check_argocd_status.py
+++ b/scripts/check_argocd_status.py
@@ -19,3 +19,4 @@ def main():

 if __name__ == '__main__':
    main()
+
--- a/scripts/check_auth_links.py
+++ b/scripts/check_auth_links.py
@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT provider_id, application_id FROM authentik_core_application_backchannel_providers;\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Backchannel Providers:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = """
+k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"SELECT * FROM authentik_providers_proxy_proxyprovider;\\""
+"""
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Proxy Providers:")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+
--- a/scripts/check_auth_tables.py
+++ b/scripts/check_auth_tables.py
@@ -0,0 +1,27 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT name, slug FROM authentik_core_application;\"'"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Apps:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT name, external_host FROM authentik_providers_proxy_proxyprovider;\"'"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Proxy Providers:")
+print(stdout.read().decode('utf-8'))
+
+cmd3 = "k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \"SELECT name, client_id FROM authentik_providers_oauth2_oauth2provider;\"'"
+stdin, stdout, stderr = c.exec_command(cmd3)
+print("OAuth2/OIDC Providers:")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+
--- a/scripts/check_authentik_apps.py
+++ b/scripts/check_authentik_apps.py
@@ -0,0 +1,27 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, authorization_flow_id FROM authentik_core_application;'"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("Authentik Applications:")
+print(stdout.read().decode('utf-8'))
+
+cmd2 = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, external_host FROM authentik_providers_proxy_proxyprovider;'"
+stdin, stdout, stderr = c.exec_command(cmd2)
+print("Authentik Proxy Providers:")
+print(stdout.read().decode('utf-8'))
+
+cmd3 = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, client_id FROM authentik_providers_oauth2_oauth2provider;'"
+stdin, stdout, stderr = c.exec_command(cmd3)
+print("Authentik OAuth2/OIDC Providers:")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+
--- a/scripts/check_authentik_apps_full.py
+++ b/scripts/check_authentik_apps_full.py
@@ -0,0 +1,31 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/core/applications/?page_size=100", headers=headers)
+with urllib.request.urlopen(req, context=ctx) as resp:
+    data = json.loads(resp.read().decode())
+    print(f"Total Applications: {len(data['results'])}")
+    for app in sorted(data['results'], key=lambda x: x['name']):
+        slug = app['slug']
+        if any(k in slug for k in ['dns', 'pihole', 'technitium', 'syno', 'nas', 'semaphore']):
+            print(f"- Name: {app['name']}")
+            print(f"  Slug: {app['slug']}")
+            print(f"  Group: {app.get('group')}")
+            print(f"  Launch URL: {app.get('meta_launch_url')}")
+            print(f"  Icon: {app.get('meta_icon')}")
+            print()
+
--- a/scripts/check_authentik_events.py
+++ b/scripts/check_authentik_events.py
@@ -0,0 +1,36 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+AK_BASE = "https://authentik.snarfnet.net"
+AK_TOKEN = "SnarfNetPermanentAutomationToken2026"
+
+headers = {
+    "Authorization": f"Bearer {AK_TOKEN}",
+    "Content-Type": "application/json",
+    "Accept": "application/json"
+}
+
+# Fetch recent events
+req = urllib.request.Request(f"{AK_BASE}/api/v3/events/events/?page_size=20", headers=headers)
+try:
+    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+        data = json.loads(r.read().decode())
+        for ev in data.get("results", []):
+            action = ev.get("action")
+            user = ev.get("user", {}).get("username")
+            context = ev.get("context", {})
+            created = ev.get("created")
+            print(f"[{created}] Action: {action} | User: {user}")
+            if "message" in context or "redirect_uri" in str(context) or "error" in str(context) or "client_id" in str(context):
+                print(f"   Context: {json.dumps(context, indent=2)}")
+except Exception as e:
+    print(f"Error querying Authentik: {e}")
+
--- a/scripts/check_authentik_ingress.py
+++ b/scripts/check_authentik_ingress.py
@@ -0,0 +1,20 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get middleware.traefik.io,middleware.traefik.containo.us -A 2>/dev/null")
+print("=== TRAEFIK MIDDLEWARES ===")
+print(stdout.read().decode('utf-8'))
+
+stdin, stdout, stderr = c.exec_command("k3s kubectl get ingress -A")
+print("=== ALL INGRESSES ===")
+print(stdout.read().decode('utf-8'))
+
+c.close()
+
--- a/scripts/check_authentik_logs.py
+++ b/scripts/check_authentik_logs.py
@@ -0,0 +1,19 @@
+import paramiko
+
+key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=key_path, timeout=10)
+
+# Check Authentik server logs
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl logs -n infra deployment/authentik-server --tail=50")
+print("Authentik server logs:")
+print(stdout.read().decode())
+
+# Check Authentik outpost logs if any
+stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n infra")
+print("Infra pods on Aerodrome:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_authentik_outposts.py
+++ b/scripts/check_authentik_outposts.py
@@ -0,0 +1,20 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/outposts/instances/", headers=headers)
+with urllib.request.urlopen(req, context=ctx) as resp:
+    data = json.loads(resp.read().decode())
+    for o in data['results']:
+        print("Outpost:", o['name'], o['type'], o.get('providers'))
+
--- a/scripts/check_authentik_proxy_providers.py
+++ b/scripts/check_authentik_proxy_providers.py
@@ -0,0 +1,25 @@
+import urllib.request, ssl, json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+AK_BASE = "https://authentik.snarfnet.net"
+AK_TOKEN = "SnarfNetPermanentAutomationToken2026"
+
+def req(endpoint):
+    url = f"{AK_BASE}/api/v3/{endpoint}"
+    r = urllib.request.Request(url, headers={"Authorization": f"Bearer {AK_TOKEN}"})
+    with urllib.request.urlopen(r, timeout=10, context=ctx) as res:
+        return json.loads(res.read().decode())
+
+print("=== Authentik Providers (Proxy) ===")
+providers = req("providers/proxy/")
+for p in providers.get("results", []):
+    print(p["pk"], p["name"], p.get("internal_host"), p.get("external_host"), p.get("mode"))
+
+print("\n=== Authentik Outposts ===")
+outposts = req("outposts/instances/")
+for o in outposts.get("results", []):
+    print(o["pk"], o["name"], o["type"], [p for p in o.get("providers", [])])
+
--- a/scripts/check_authentik_semaphore.py
+++ b/scripts/check_authentik_semaphore.py
@@ -0,0 +1,26 @@
+import urllib.request
+import ssl
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/providers/oauth2/", headers=headers)
+with urllib.request.urlopen(req, context=ctx) as resp:
+    data = json.loads(resp.read().decode())
+    for p in data['results']:
+        if 'semaphore' in p['name'].lower():
+            print(f"Provider ID: {p['pk']}, Name: {p['name']}")
+            print(f"Redirect URIs: {p.get('redirect_uris')}")
+            print(f"Client ID: {p.get('client_id')}")
+
--- a/scripts/check_authentik_users.py
+++ b/scripts/check_authentik_users.py
@@ -0,0 +1,20 @@
+import urllib.request
+import ssl
+import json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+token = "SnarfNetPermanentAutomationToken2026"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+req = urllib.request.Request("https://authentik.snarfnet.net/api/v3/core/users/?page_size=20", headers=headers)
+with urllib.request.urlopen(req, context=ctx) as resp:
+    data = json.loads(resp.read().decode())
+    for u in data['results']:
+        print(f"User: {u['username']}, Email: {u.get('email')}, Name: {u.get('name')}")
+
--- a/scripts/check_awx_monitor.py
+++ b/scripts/check_awx_monitor.py
@@ -0,0 +1,16 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, url, port, accepted_statuscodes_json FROM monitor WHERE name LIKE '%AWX%';" """))
+print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM heartbeat WHERE monitor_id IN (SELECT id FROM monitor WHERE name LIKE '%AWX%') ORDER BY id DESC LIMIT 5;" """))
+
+c.close()
+
--- a/scripts/check_blkid.py
+++ b/scripts/check_blkid.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("blkid")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/scripts/check_boot_ctl03.py
+++ b/scripts/check_boot_ctl03.py
@@ -0,0 +1,14 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
+stdin, stdout, stderr = client.exec_command("ls -la /boot; ls -la /boot/firmware 2>/dev/null | head -n 20")
+print(stdout.read().decode('utf-8'))
+client.close()
+
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Snarf	336b7759d0	feat: add infrastructure automation and diagnostic scripts for Zabbix, Nginx, and Authentik	2026-11-12 20:18:37 -05:00
Snarf	68c108cfe8	docs(topology): regenerate high-res PNG diagram to reflect latest layout, hardware and storage metrics	2026-11-12 17:13:51 -05:00
Snarf	9ebd48af0c	chore: update homelab-gitops-repo submodule reference	2026-11-12 17:09:51 -05:00
Snarf	113ebdc7c5	feat: add automation and inspection scripts for Nginx Proxy Manager and Authentik integration	2026-11-12 17:08:14 -05:00
Snarf	5a70460e89	feat: add homelab deployment configs, documentation, and automation scripts	2026-11-12 16:57:08 -05:00
Snarf	1228b957de	chore: remove empty file	2026-11-08 15:03:27 -05:00
Snarf	3c373351b7	fix: remove empty file diff	2026-11-08 14:57:08 -05:00
Snarf	e4962b7334	chore: remove unused file	2026-11-08 14:17:31 -05:00
Snarf	7f07fa5a89	chore: remove empty file	2026-11-08 13:51:41 -05:00
Snarf	fc42bd6f94	feat: add DNS setup blueprint and infrastructure automation scripts	2026-11-08 09:28:06 -05:00
Snarf	17e60354a1	feat(idrac): add iDRAC cluster check script and update redirection helpers	2026-11-07 21:09:41 -05:00
Snarf	1d4dfb594b	feat(dns): point snarfhome.snarfnet.net to 14.4.4.20 and remove Nabu Casa redirection	2026-11-07 18:21:46 -05:00
Snarf	42bfb43b65	feat: add scripts to inspect snarfhome proxy and redirection configurations via SSH	2026-11-07 16:56:11 -05:00
Snarf	bbcf656a3d	feat: add automation scripts for inspecting proxy providers and updating Nginx configurations	2026-11-07 15:41:04 -05:00
Snarf	e0ebc120ee	Configure VaultConfigurator direct proxy in NPM and update topology docs	2026-11-07 15:19:19 -05:00
Snarf	40e504c2fe	feat(vault): add vault.snarfnet.net proxy host and topology docs	2026-11-07 14:41:42 -05:00
Snarf	34cfece0b0	feat: add diagnostic and maintenance scripts for Pi-hole, Technitium, and Nginx Proxy Manager	2026-11-07 13:43:11 -05:00
Snarf	4e8f93173e	docs(topology): update AI stack hostnames to snnpud4lnxops01/snnpud4lnxweb01 and prune obsolete shipyardcontrol	2026-11-07 13:37:27 -05:00
Snarf	1fe16429a4	feat: add script to check pihole-FTL database integrity and query count via SSH	2026-11-07 13:02:12 -05:00
Snarf	ac1395486d	feat: add script to apply DNS manifests via SSH to cluster nodes	2026-11-07 12:44:53 -05:00
Snarf	a07cecb6f3	feat: add script to apply DNS manifests via SSH and kubectl	2026-11-07 12:42:44 -05:00
Snarf	ffc9ab8ab0	feat: add diagnostic and traffic policy management scripts for DNS and Pi-hole	2026-11-07 12:42:31 -05:00
Snarf	d4b367794c	feat: add OpenCanary honeypot documentation and management scripts	2026-11-07 00:32:19 -05:00
Snarf	3871d04d48	feat(security): deploy OpenCanary honeypot on VLAN 50 with Gotify and Discord alerting	2026-11-07 00:22:51 -05:00
Snarf	0ed293b6ea	feat: add infrastructure diagnostic and monitoring scripts	2026-11-06 23:12:35 -05:00
Snarf	d5c2f6eae5	feat: add diagnostic and management scripts for DNS, NPM, and cluster infrastructure	2026-11-06 19:08:19 -05:00
Snarf	8e1c5f1e66	feat: add automation scripts for NAS fixes, Authentik monitoring, and Technitium DNS management	2026-11-06 18:49:24 -05:00
Snarf	1965d37687	feat: add helper scripts for infrastructure management and troubleshooting	2026-11-06 15:24:05 -05:00
Snarf	aea8608f3d	feat: add infrastructure audit and diagnostic helper scripts	2026-11-06 12:37:49 -05:00
Snarf	06c203485f	feat: add scripts to apply pihole manifests and update authentik launch URLs	2026-11-06 09:06:17 -05:00
Snarf	b12c143791	refactor: remove unused file	2026-11-06 09:04:08 -05:00
Snarf	d9ca093155	feat: add automation scripts for checking and managing remote DNS and SSO services	2026-11-06 09:03:28 -05:00
Snarf	1b8c862b10	feat: add various infrastructure and service diagnostic scripts	2026-11-06 09:02:19 -05:00
Snarf	00f22f33af	feat: add diagnostic and inspection utility scripts for infrastructure and services	2026-11-06 00:08:53 -05:00
Snarf	47acedb455	fix: remove obsolete file	2026-11-05 21:40:37 -05:00
Snarf	fffb446206	feat: add cluster management and audit utility scripts	2026-11-05 21:40:22 -05:00
Snarf	fa969fc1d3	docs: add network blueprint for high-availability DNS, honeypot, and UDM firewall hardening	2026-11-05 11:44:43 -05:00
Snarf	f1c197422d	docs: add DNS setup plan documentation	2026-11-05 11:20:57 -05:00
Snarf	17ca433d0c	feat: add diagnostic scripts for gateway and DNS investigation	2026-11-05 11:17:12 -05:00
Snarf	88191155d2	feat: add administrative and deployment scripts for infrastructure management	2026-11-05 11:10:07 -05:00
Snarf	cbdb37a453	feat: add script to read and parse transcript jsonl steps	2026-11-05 10:17:02 -05:00
Snarf	1fce3f2488	feat: add infrastructure health check and management automation scripts	2026-11-05 10:13:07 -05:00
Snarf	14fac7151f	feat: add scripts to audit and troubleshoot spaceport kernels and cluster nodes	2026-11-05 07:28:22 -05:00
Snarf	b974980d08	feat: add script to check cattle-system pods and events on spaceport server	2026-11-05 06:50:56 -05:00
Snarf	5b955fcbc3	feat: add housekeeping and cleanup scripts for cluster maintenance	2026-11-05 06:50:35 -05:00
Snarf	80738fd8c1	feat: add script to check Rancher cluster and cattle-system status via SSH	2026-11-04 23:32:54 -05:00
Snarf	11749df634	feat: add diagnostic and management scripts for Rancher k3s deployment	2026-11-04 23:27:34 -05:00
Snarf	1e7935e789	feat: add scripts for automating and checking Rancher upgrades via SSH	2026-11-04 23:10:28 -05:00
Snarf	706fccacb1	test: add dry-run script for rancher 2.14.3 helm upgrade	2026-11-04 22:34:52 -05:00
Snarf	114b619c1f	test: add SSH dry-run scripts for Rancher chart versions 2.11.3, 2.12.3, and 2.13.3	2026-11-04 22:25:05 -05:00
Snarf	45f9b557b9	test: add dry-run script for rancher 2.10.3 helm chart deployment	2026-11-04 21:58:01 -05:00
Snarf	07dae0c73e	feat: add scripts for checking rancher clusters and testing helm dry-run	2026-11-04 21:48:30 -05:00
Snarf	93e13c7482	feat: add scripts for testing and automating Rancher upgrades and chart checks	2026-11-04 21:35:52 -05:00
Snarf	8b21087c89	feat: add scripts for testing and automating Rancher upgrades and chart checks	2026-11-04 21:34:42 -05:00
Snarf	ba47f6a384	feat: add scripts to inspect and check Rancher deployments via SSH	2026-11-04 21:26:45 -05:00
Snarf	1b378b29f3	feat: add cluster auditing, management, and upgrade scripts	2026-11-04 17:58:07 -05:00
Snarf	8117326b14	feat(k3s): upgrade Aerodrome control plane and workers to v1.31.4+k3s1 with rolling drain and safety gates	2026-11-03 16:45:35 -05:00
Snarf	4f0fa3fe55	fix(patching): add debconf non-interactive pre-seeding, cordon/drain, and storage settling to weekly OS patching	2026-11-03 15:59:08 -05:00
Snarf	42f669266e	chore: remove unused file	2026-11-03 14:32:06 -05:00
Snarf	5a583d1db9	feat(pillar-3): tune longhorn settings for power resilience on aerodrome and spaceport	2026-11-03 10:50:37 -05:00
Snarf	5cb00c04a9	feat(pillar-4): complete dual-npm keepalived HA cluster on VIP 140.44.4.70	2026-11-03 01:05:05 -05:00
Snarf	9391b3f2a1	feat(vip): lock MetalLB LoadBalancer VIPs across Aerodrome and Spaceport (Pillar 2 completed)	2026-11-02 23:28:30 -05:00
Snarf	3708404838	feat: add infrastructure automation and management scripts	2026-11-02 22:09:05 -05:00
Snarf	d3e2e1f291	feat(power): complete Phase 7 Power Resilience & UPS Automation deployment and verification	2026-11-02 21:58:49 -05:00