docs: modernize power resilience plan and shutdown orchestration with NFS-safe tiering, direct Gotify alerts, and Longhorn clean detach

scripts/homelab_power_shutdown.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# /usr/local/bin/homelab_power_shutdown.sh
+# Staged graceful homelab shutdown orchestrator for Snarfnet
+set -uo pipefail
+
+log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [POWER-SHUTDOWN] $*" | tee -a /var/log/power-shutdown.log; }
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=3 -o BatchMode=yes"
+
+log "=== INITIATING HOMELAB TIERED SHUTDOWN ==="
+
+# ------------------------------------------------------------------
+# TIER 1: Non-Essential Compute & AI Pis
+# ------------------------------------------------------------------
+log "Stage 1: Halting AI Raspberry Pi stack (140.44.4.7, 140.44.4.8)..."
+ssh $SSH_OPTS root@140.44.4.7 "shutdown -h now" 2>/dev/null || true
+ssh $SSH_OPTS root@140.44.4.8 "shutdown -h now" 2>/dev/null || true
+
+# ------------------------------------------------------------------
+# TIER 2: Docker Swarm NFS Workloads (NPM & Peanut)
+# ------------------------------------------------------------------
+log "Stage 2: Stopping Swarm services mounting Synology NFS (NPM, Peanut)..."
+ssh $SSH_OPTS root@140.44.4.71 "docker service scale nginx-proxy-manager_app=0 nginx-proxy-manager_backup=0 peanut_peanut=0" 2>/dev/null || true
+sleep 5
+
+# ------------------------------------------------------------------
+# TIER 3: Harvester Guest VMs & Kubernetes Worker Nodes
+# ------------------------------------------------------------------
+log "Stage 3: Shutting down Harvester guest VMs and K3s workers..."
+# Gracefully signal Harvester VMs
+ssh $SSH_OPTS root@140.44.4.81 "virsh list --name | xargs -r -n1 virsh shutdown" 2>/dev/null || true
+
+# Stop K3s agent on Aerodrome workers (clean Longhorn PVC detach)
+for ip in 140.44.4.22 140.44.4.23 140.44.4.24 140.44.4.25; do
+    ssh $SSH_OPTS root@$ip "systemctl stop k3s-agent && sync && shutdown -h now" 2>/dev/null || true
+done
+
+# Stop K3s agent on Spaceport workers
+for ip in 140.44.4.51 140.44.4.52 140.44.4.53 140.44.4.54 140.44.4.55 140.44.4.56; do
+    ssh $SSH_OPTS root@$ip "systemctl stop k3s-agent && sync && shutdown -h now" 2>/dev/null || true
+done
+sleep 15
+
+# ------------------------------------------------------------------
+# TIER 4: Storage Arrays (Synology DSM Safe Mode)
+# ------------------------------------------------------------------
+log "Stage 4: Triggering Synology NAS Safe Mode (14.10.10.71, 14.10.10.72)..."
+ssh $SSH_OPTS admin@14.10.10.71 "synopoweroff" 2>/dev/null || true
+ssh $SSH_OPTS admin@14.10.10.72 "synopoweroff" 2>/dev/null || true
+sleep 20
+
+# ------------------------------------------------------------------
+# TIER 5: Control Planes & Swarm Managers
+# ------------------------------------------------------------------
+log "Stage 5: Halting K3s control planes and Swarm managers..."
+# Spaceport Masters (140.44.4.41-44)
+for ip in 140.44.4.41 140.44.4.42 140.44.4.43 140.44.4.44; do
+    ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save && systemctl stop k3s && shutdown -h now" 2>/dev/null || true
+done
+
+# Aerodrome Masters (140.44.4.11-13)
+for ip in 140.44.4.11 140.44.4.12 140.44.4.13; do
+    ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save && systemctl stop k3s && shutdown -h now" 2>/dev/null || true
+done
+
+# Swarm Managers (140.44.4.71, 72, 74)
+for ip in 140.44.4.71 140.44.4.72 140.44.4.74; do
+    ssh $SSH_OPTS root@$ip "sync && shutdown -h now" 2>/dev/null || true
+done
+
+# Harvester Baremetal Hypervisors (140.44.4.81-83)
+for ip in 140.44.4.81 140.44.4.82 140.44.4.83; do
+    ssh $SSH_OPTS root@$ip "sync && shutdown -h now" 2>/dev/null || true
+done
+
+# ------------------------------------------------------------------
+# TIER 6: Master Node Self-Cutoff (FSD)
+# ------------------------------------------------------------------
+log "Stage 6: Executing final local halt and arming killpower..."
+touch /etc/killpower
+/sbin/shutdown -h +0

scripts/nut_notify.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# /usr/local/bin/nut-notify.sh
+# Emergency power notification script for Network UPS Tools on Snarfnet
+set -euo pipefail
+
+EVENT_TYPE="${1:-$NOTIFYTYPE}"
+UPS_NAME="${UPSNAME:-snarf-ups}"
+TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+
+# Internal Gotify VIP endpoint (bypasses Authentik Forward Auth and NPM)
+GOTIFY_INTERNAL_URL="http://140.44.4.70:8090/message"
+GOTIFY_APP_TOKEN="${GOTIFY_APP_TOKEN:-A4uN4fEmErGeNcYtOkEn}"
+
+send_gotify() {
+    local title="$1"
+    local message="$2"
+    local priority="$3"
+    
+    curl -s -m 5 -X POST "${GOTIFY_INTERNAL_URL}?token=${GOTIFY_APP_TOKEN}" \
+        -H "Content-Type: application/json" \
+        -d "{\"title\": \"${title}\", \"message\": \"${message}\", \"priority\": ${priority}}" >/dev/null 2>&1 || true
+}
+
+case "$EVENT_TYPE" in
+    ONBATT)
+        send_gotify "⚡ Grid Power Lost" "[$TIMESTAMP] Homelab is running on UPS battery (${UPS_NAME}). Staged shutdown countdown started." 8
+        logger -t nut-notify "Grid power failure detected - running on UPS battery."
+        ;;
+    ONLINE)
+        send_gotify "✅ Grid Power Restored" "[$TIMESTAMP] Utility AC power restored to ${UPS_NAME}. Battery recharging." 5
+        logger -t nut-notify "AC utility power restored."
+        ;;
+    LOWBATT)
+        send_gotify "🚨 UPS Battery Critical" "[$TIMESTAMP] Battery level critical on ${UPS_NAME}! Final Tier 4-6 shutdown initiated." 10
+        logger -t nut-notify "CRITICAL: UPS battery low. Enforcing emergency shutdown."
+        ;;
+    FSD)
+        send_gotify "🛑 Forced Shutdown Engaged" "[$TIMESTAMP] NUT Master has commanded immediate power cutoff across all clusters." 10
+        logger -t nut-notify "FSD signal received. Halting systems."
+        ;;
+    *)
+        logger -t nut-notify "Unhandled UPS event: ${EVENT_TYPE}"
+        ;;
+esac