Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(power): complete Phase 7 Power Resilience & UPS Automation deployment and verification

Browse Source
This commit is contained in:
Snarf
2026-11-02 21:58:49 -05:00
parent 7ca3d5acb8
commit d3e2e1f291
8 changed files with 216 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
scripts/homelab_power_shutdown.sh
View File

@@ -4,37 +4,34 @@
set -uo pipefail
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [POWER-SHUTDOWN] $*" | tee -a /var/log/power-shutdown.log; }
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=3 -o BatchMode=yes"
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=4 -o BatchMode=yes -i /root/.ssh/id_ed25519_snarfnet"
log "=== INITIATING HOMELAB TIERED SHUTDOWN ==="
# ------------------------------------------------------------------
# TIER 1: Non-Essential Compute & AI Pis
# ------------------------------------------------------------------
log "Stage 1: Halting AI Raspberry Pi stack (140.44.4.7, 140.44.4.8)..."
log "Tier 1: Halting AI Raspberry Pi stack (140.44.4.7, 140.44.4.8)..."
ssh $SSH_OPTS root@140.44.4.7 "shutdown -h now" 2>/dev/null || true
ssh $SSH_OPTS root@140.44.4.8 "shutdown -h now" 2>/dev/null || true
# ------------------------------------------------------------------
# TIER 2: Docker Swarm NFS Workloads (NPM & Peanut)
# TIER 2: Docker Swarm NFS Workloads (NPM, Peanut, Homebox)
# ------------------------------------------------------------------
log "Stage 2: Stopping Swarm services mounting Synology NFS (NPM, Peanut)..."
ssh $SSH_OPTS root@140.44.4.71 "docker service scale nginx-proxy-manager_app=0 nginx-proxy-manager_backup=0 peanut_peanut=0" 2>/dev/null || true
log "Tier 2: Stopping Swarm services mounting Synology NFS..."
ssh $SSH_OPTS root@140.44.4.71 "docker service scale nginx-proxy-manager_app=0 nginx-proxy-manager_backup=0 peanut_peanut=0 homebox_homebox=0" 2>/dev/null || true
sleep 5
# ------------------------------------------------------------------
# TIER 3: Harvester Guest VMs & Kubernetes Worker Nodes
# TIER 3: Kubernetes Worker Nodes (Aerodrome & Spaceport)
# ------------------------------------------------------------------
log "Stage 3: Shutting down Harvester guest VMs and K3s workers..."
# Gracefully signal Harvester VMs
ssh $SSH_OPTS root@140.44.4.81 "virsh list --name | xargs -r -n1 virsh shutdown" 2>/dev/null || true
# Stop K3s agent on Aerodrome workers (clean Longhorn PVC detach)
for ip in 140.44.4.22 140.44.4.23 140.44.4.24 140.44.4.25; do
log "Tier 3: Gracefully stopping K3s agent and halting worker nodes..."
# Aerodrome Workers (wrk01 - wrk07)
for ip in 140.44.4.21 140.44.4.22 140.44.4.23 140.44.4.24 140.44.4.25 140.44.4.26 140.44.4.27; do
ssh $SSH_OPTS root@$ip "systemctl stop k3s-agent && sync && shutdown -h now" 2>/dev/null || true
done
# Stop K3s agent on Spaceport workers
# Spaceport Workers (wrk01 - wrk06)
for ip in 140.44.4.51 140.44.4.52 140.44.4.53 140.44.4.54 140.44.4.55 140.44.4.56; do
ssh $SSH_OPTS root@$ip "systemctl stop k3s-agent && sync && shutdown -h now" 2>/dev/null || true
done
@@ -43,39 +40,34 @@ sleep 15
# ------------------------------------------------------------------
# TIER 4: Storage Arrays (Synology DSM Safe Mode)
# ------------------------------------------------------------------
log "Stage 4: Triggering Synology NAS Safe Mode (14.10.10.71, 14.10.10.72)..."
ssh $SSH_OPTS admin@14.10.10.71 "synopoweroff" 2>/dev/null || true
ssh $SSH_OPTS admin@14.10.10.72 "synopoweroff" 2>/dev/null || true
sleep 20
# Synology NAS (14.10.10.71, 14.10.10.72) automatically enters DSM Safe Mode
# via native NUT slave daemon on receiving LOWBATT/FSD from 14.4.4.44.
log "Tier 4: Synology NAS Safe Mode engaged via NUT slave protocol."
sleep 10
# ------------------------------------------------------------------
# TIER 5: Control Planes & Swarm Managers
# TIER 5: Kubernetes Control Planes & Swarm Managers
# ------------------------------------------------------------------
log "Stage 5: Halting K3s control planes and Swarm managers..."
# Spaceport Masters (140.44.4.41-44)
log "Tier 5: Halting K3s control planes and Swarm managers..."
# Spaceport Masters (ctl01 - ctl04)
for ip in 140.44.4.41 140.44.4.42 140.44.4.43 140.44.4.44; do
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save && systemctl stop k3s && shutdown -h now" 2>/dev/null || true
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save --name pre-shutdown-snapshot 2>/dev/null || true; systemctl stop k3s && sync && shutdown -h now" 2>/dev/null || true
done
# Aerodrome Masters (140.44.4.11-13)
# Aerodrome Masters (ctl01 - ctl03)
for ip in 140.44.4.11 140.44.4.12 140.44.4.13; do
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save && systemctl stop k3s && shutdown -h now" 2>/dev/null || true
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save --name pre-shutdown-snapshot 2>/dev/null || true; systemctl stop k3s && sync && shutdown -h now" 2>/dev/null || true
done
# Swarm Managers (140.44.4.71, 72, 74)
for ip in 140.44.4.71 140.44.4.72 140.44.4.74; do
ssh $SSH_OPTS root@$ip "sync && shutdown -h now" 2>/dev/null || true
done
# Harvester Baremetal Hypervisors (140.44.4.81-83)
for ip in 140.44.4.81 140.44.4.82 140.44.4.83; do
# Swarm Managers (swm01 - swm05)
for ip in 140.44.4.71 140.44.4.72 140.44.4.73 140.44.4.74 140.44.4.75; do
ssh $SSH_OPTS root@$ip "sync && shutdown -h now" 2>/dev/null || true
done
sleep 5
# ------------------------------------------------------------------
# TIER 6: Master Node Self-Cutoff (FSD)
# TIER 6: NUT Master Final Signal & Edge Cutoff
# ------------------------------------------------------------------
log "Stage 6: Executing final local halt and arming killpower..."
log "Tier 6: Executing final local halt and arming killpower..."
touch /etc/killpower
/sbin/shutdown -h +0