feat(power): complete Phase 7 Power Resilience & UPS Automation deployment and verification
This commit is contained in:
20
scripts/check_nut_services.py
Normal file
20
scripts/check_nut_services.py
Normal file
@@ -0,0 +1,20 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('14.4.4.44', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('systemctl status nut-monitor --no-pager')
|
||||
print("STATUS NUT-MONITOR:\n", stdout.read().decode('utf-8', errors='replace'))
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('systemctl status nut-server --no-pager')
|
||||
print("STATUS NUT-SERVER:\n", stdout.read().decode('utf-8', errors='replace'))
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
15
scripts/check_nut_users.py
Normal file
15
scripts/check_nut_users.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('14.4.4.44', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('grep -E "NOTIFY|SHUTDOWN" /etc/nut/upsmon.conf | grep -v "^#"')
|
||||
print("=== NOTIFY / SHUTDOWN IN UPSMON.CONF ===")
|
||||
print(stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
72
scripts/deploy_nut_config.py
Normal file
72
scripts/deploy_nut_config.py
Normal file
@@ -0,0 +1,72 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('14.4.4.44', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
# 1. Deploy /usr/local/bin/nut-notify.sh
|
||||
with open(r'scripts\nut_notify.sh', 'r', encoding='utf-8') as f:
|
||||
nut_notify_content = f.read()
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.file('/usr/local/bin/nut-notify.sh', 'w') as f:
|
||||
f.write(nut_notify_content)
|
||||
ssh.exec_command('chmod 755 /usr/local/bin/nut-notify.sh')
|
||||
print("Deployed /usr/local/bin/nut-notify.sh")
|
||||
|
||||
# 2. Deploy /usr/local/bin/homelab_power_shutdown.sh
|
||||
with open(r'scripts\homelab_power_shutdown.sh', 'r', encoding='utf-8') as f:
|
||||
power_shutdown_content = f.read()
|
||||
|
||||
with sftp.file('/usr/local/bin/homelab_power_shutdown.sh', 'w') as f:
|
||||
f.write(power_shutdown_content)
|
||||
ssh.exec_command('chmod 755 /usr/local/bin/homelab_power_shutdown.sh')
|
||||
print("Deployed /usr/local/bin/homelab_power_shutdown.sh")
|
||||
sftp.close()
|
||||
|
||||
# 3. Update /etc/nut/upsmon.conf
|
||||
upsmon_conf = """
|
||||
MONITOR server-closet@localhost 1 admin e3e34r4rT%T%^Y^Y primary
|
||||
MINSUPPLIES 1
|
||||
SHUTDOWNCMD "/usr/local/bin/homelab_power_shutdown.sh"
|
||||
NOTIFYCMD /usr/local/bin/nut-notify.sh
|
||||
POLLFREQ 5
|
||||
POLLFREQALERT 5
|
||||
HOSTSYNC 15
|
||||
DEADTIME 15
|
||||
POWERDOWNFLAG /etc/killpower
|
||||
RBWARNTIME 43200
|
||||
NOCOMMWARNTIME 300
|
||||
FINALDELAY 180
|
||||
|
||||
NOTIFYFLAG ONLINE SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG ONBATT SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG LOWBATT SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG FSD SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG COMMOK SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG COMMBAD SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG SHUTDOWN SYSLOG+WALL+EXEC
|
||||
NOTIFYFLAG REPLBATT SYSLOG+WALL
|
||||
NOTIFYFLAG NOCOMM SYSLOG+WALL
|
||||
"""
|
||||
# Backup existing upsmon.conf
|
||||
ssh.exec_command('cp /etc/nut/upsmon.conf /etc/nut/upsmon.conf.bak')
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('cat > /etc/nut/upsmon.conf')
|
||||
stdin.write(upsmon_conf.strip() + "\n")
|
||||
stdin.channel.shutdown_write()
|
||||
print("Updated /etc/nut/upsmon.conf")
|
||||
|
||||
# Restart nut-client / nut-monitor
|
||||
stdin, stdout, stderr = ssh.exec_command('systemctl restart nut-client || systemctl restart nut-monitor')
|
||||
print("Restart service:", stdout.read().decode(), stderr.read().decode())
|
||||
|
||||
# Check status
|
||||
stdin, stdout, stderr = ssh.exec_command('systemctl is-active nut-client || systemctl is-active nut-monitor')
|
||||
print("Service is-active:", stdout.read().decode().strip())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -4,37 +4,34 @@
|
||||
set -uo pipefail
|
||||
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [POWER-SHUTDOWN] $*" | tee -a /var/log/power-shutdown.log; }
|
||||
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=3 -o BatchMode=yes"
|
||||
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=4 -o BatchMode=yes -i /root/.ssh/id_ed25519_snarfnet"
|
||||
|
||||
log "=== INITIATING HOMELAB TIERED SHUTDOWN ==="
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# TIER 1: Non-Essential Compute & AI Pis
|
||||
# ------------------------------------------------------------------
|
||||
log "Stage 1: Halting AI Raspberry Pi stack (140.44.4.7, 140.44.4.8)..."
|
||||
log "Tier 1: Halting AI Raspberry Pi stack (140.44.4.7, 140.44.4.8)..."
|
||||
ssh $SSH_OPTS root@140.44.4.7 "shutdown -h now" 2>/dev/null || true
|
||||
ssh $SSH_OPTS root@140.44.4.8 "shutdown -h now" 2>/dev/null || true
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# TIER 2: Docker Swarm NFS Workloads (NPM & Peanut)
|
||||
# TIER 2: Docker Swarm NFS Workloads (NPM, Peanut, Homebox)
|
||||
# ------------------------------------------------------------------
|
||||
log "Stage 2: Stopping Swarm services mounting Synology NFS (NPM, Peanut)..."
|
||||
ssh $SSH_OPTS root@140.44.4.71 "docker service scale nginx-proxy-manager_app=0 nginx-proxy-manager_backup=0 peanut_peanut=0" 2>/dev/null || true
|
||||
log "Tier 2: Stopping Swarm services mounting Synology NFS..."
|
||||
ssh $SSH_OPTS root@140.44.4.71 "docker service scale nginx-proxy-manager_app=0 nginx-proxy-manager_backup=0 peanut_peanut=0 homebox_homebox=0" 2>/dev/null || true
|
||||
sleep 5
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# TIER 3: Harvester Guest VMs & Kubernetes Worker Nodes
|
||||
# TIER 3: Kubernetes Worker Nodes (Aerodrome & Spaceport)
|
||||
# ------------------------------------------------------------------
|
||||
log "Stage 3: Shutting down Harvester guest VMs and K3s workers..."
|
||||
# Gracefully signal Harvester VMs
|
||||
ssh $SSH_OPTS root@140.44.4.81 "virsh list --name | xargs -r -n1 virsh shutdown" 2>/dev/null || true
|
||||
|
||||
# Stop K3s agent on Aerodrome workers (clean Longhorn PVC detach)
|
||||
for ip in 140.44.4.22 140.44.4.23 140.44.4.24 140.44.4.25; do
|
||||
log "Tier 3: Gracefully stopping K3s agent and halting worker nodes..."
|
||||
# Aerodrome Workers (wrk01 - wrk07)
|
||||
for ip in 140.44.4.21 140.44.4.22 140.44.4.23 140.44.4.24 140.44.4.25 140.44.4.26 140.44.4.27; do
|
||||
ssh $SSH_OPTS root@$ip "systemctl stop k3s-agent && sync && shutdown -h now" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Stop K3s agent on Spaceport workers
|
||||
# Spaceport Workers (wrk01 - wrk06)
|
||||
for ip in 140.44.4.51 140.44.4.52 140.44.4.53 140.44.4.54 140.44.4.55 140.44.4.56; do
|
||||
ssh $SSH_OPTS root@$ip "systemctl stop k3s-agent && sync && shutdown -h now" 2>/dev/null || true
|
||||
done
|
||||
@@ -43,39 +40,34 @@ sleep 15
|
||||
# ------------------------------------------------------------------
|
||||
# TIER 4: Storage Arrays (Synology DSM Safe Mode)
|
||||
# ------------------------------------------------------------------
|
||||
log "Stage 4: Triggering Synology NAS Safe Mode (14.10.10.71, 14.10.10.72)..."
|
||||
ssh $SSH_OPTS admin@14.10.10.71 "synopoweroff" 2>/dev/null || true
|
||||
ssh $SSH_OPTS admin@14.10.10.72 "synopoweroff" 2>/dev/null || true
|
||||
sleep 20
|
||||
# Synology NAS (14.10.10.71, 14.10.10.72) automatically enters DSM Safe Mode
|
||||
# via native NUT slave daemon on receiving LOWBATT/FSD from 14.4.4.44.
|
||||
log "Tier 4: Synology NAS Safe Mode engaged via NUT slave protocol."
|
||||
sleep 10
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# TIER 5: Control Planes & Swarm Managers
|
||||
# TIER 5: Kubernetes Control Planes & Swarm Managers
|
||||
# ------------------------------------------------------------------
|
||||
log "Stage 5: Halting K3s control planes and Swarm managers..."
|
||||
# Spaceport Masters (140.44.4.41-44)
|
||||
log "Tier 5: Halting K3s control planes and Swarm managers..."
|
||||
# Spaceport Masters (ctl01 - ctl04)
|
||||
for ip in 140.44.4.41 140.44.4.42 140.44.4.43 140.44.4.44; do
|
||||
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save && systemctl stop k3s && shutdown -h now" 2>/dev/null || true
|
||||
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save --name pre-shutdown-snapshot 2>/dev/null || true; systemctl stop k3s && sync && shutdown -h now" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Aerodrome Masters (140.44.4.11-13)
|
||||
# Aerodrome Masters (ctl01 - ctl03)
|
||||
for ip in 140.44.4.11 140.44.4.12 140.44.4.13; do
|
||||
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save && systemctl stop k3s && shutdown -h now" 2>/dev/null || true
|
||||
ssh $SSH_OPTS root@$ip "k3s etcd-snapshot save --name pre-shutdown-snapshot 2>/dev/null || true; systemctl stop k3s && sync && shutdown -h now" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Swarm Managers (140.44.4.71, 72, 74)
|
||||
for ip in 140.44.4.71 140.44.4.72 140.44.4.74; do
|
||||
ssh $SSH_OPTS root@$ip "sync && shutdown -h now" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Harvester Baremetal Hypervisors (140.44.4.81-83)
|
||||
for ip in 140.44.4.81 140.44.4.82 140.44.4.83; do
|
||||
# Swarm Managers (swm01 - swm05)
|
||||
for ip in 140.44.4.71 140.44.4.72 140.44.4.73 140.44.4.74 140.44.4.75; do
|
||||
ssh $SSH_OPTS root@$ip "sync && shutdown -h now" 2>/dev/null || true
|
||||
done
|
||||
sleep 5
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# TIER 6: Master Node Self-Cutoff (FSD)
|
||||
# TIER 6: NUT Master Final Signal & Edge Cutoff
|
||||
# ------------------------------------------------------------------
|
||||
log "Stage 6: Executing final local halt and arming killpower..."
|
||||
log "Tier 6: Executing final local halt and arming killpower..."
|
||||
touch /etc/killpower
|
||||
/sbin/shutdown -h +0
|
||||
|
||||
|
||||
27
scripts/inspect_nut_master.py
Normal file
27
scripts/inspect_nut_master.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('14.4.4.44', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== UPSC METRICS ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('upsc server-closet@127.0.0.1')
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("=== NUT.CONF ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('grep -v "^#" /etc/nut/nut.conf | grep -v "^$"')
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("=== UPS.CONF ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('grep -v "^#" /etc/nut/ups.conf | grep -v "^$"')
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("=== UPSMON.CONF ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('grep -v "^#" /etc/nut/upsmon.conf | grep -v "^$"')
|
||||
print(stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -4,7 +4,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
EVENT_TYPE="${1:-$NOTIFYTYPE}"
|
||||
UPS_NAME="${UPSNAME:-snarf-ups}"
|
||||
UPS_NAME="${UPSNAME:-server-closet}"
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Internal Gotify VIP endpoint (bypasses Authentik Forward Auth and NPM)
|
||||
@@ -20,11 +20,11 @@ send_gotify() {
|
||||
local priority="$3"
|
||||
local color="${4:-16711680}"
|
||||
|
||||
# Primary: Send to Gotify (which pushes to mobile devices + forwards to Discord ups-power webhook)
|
||||
# Primary: Send to Gotify (which pushes to mobile devices + forwards to Discord #ups-power)
|
||||
if ! curl -sf -m 5 -X POST "${GOTIFY_INTERNAL_URL}?token=${GOTIFY_APP_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"${title}\", \"message\": \"${message}\", \"priority\": ${priority}}" >/dev/null 2>&1; then
|
||||
# Fallback: If Gotify/Shipyard is already powered down (Tier 4+), send directly to Discord via UDM Pro (Tier 6)
|
||||
# Fallback: If Gotify/Shipyard is already powered down (Tier 4+), send directly to Discord via UDM Pro
|
||||
curl -sf -m 5 -X POST "${DISCORD_UPS_WEBHOOK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\": \"SnarfNet NUT Master (Direct Fallback)\", \"embeds\": [{\"title\": \"${title}\", \"description\": \"${message}\", \"color\": ${color}}]}" >/dev/null 2>&1 || true
|
||||
@@ -41,15 +41,26 @@ case "$EVENT_TYPE" in
|
||||
logger -t nut-notify "AC utility power restored."
|
||||
;;
|
||||
LOWBATT)
|
||||
send_gotify "🚨 UPS Battery Critical" "[$TIMESTAMP] Battery level critical on ${UPS_NAME}! Final Tier 4-6 shutdown initiated." 10 15158332
|
||||
send_gotify "🚨 UPS Battery Critical" "[$TIMESTAMP] Battery level critical on ${UPS_NAME}! Final Tier 3-6 shutdown initiated." 10 15158332
|
||||
logger -t nut-notify "CRITICAL: UPS battery low. Enforcing emergency shutdown."
|
||||
;;
|
||||
FSD)
|
||||
send_gotify "🛑 Forced Shutdown Engaged" "[$TIMESTAMP] NUT Master has commanded immediate power cutoff across all clusters." 10 10038562
|
||||
logger -t nut-notify "FSD signal received. Halting systems."
|
||||
;;
|
||||
COMMOK)
|
||||
send_gotify "🔌 UPS Communication Restored" "[$TIMESTAMP] Communication with ${UPS_NAME} established." 4 3066993
|
||||
logger -t nut-notify "UPS communication restored."
|
||||
;;
|
||||
COMMBAD)
|
||||
send_gotify "⚠️ UPS Communication Lost" "[$TIMESTAMP] Lost communication with ${UPS_NAME}." 8 16753920
|
||||
logger -t nut-notify "UPS communication lost."
|
||||
;;
|
||||
SHUTDOWN)
|
||||
send_gotify "🛑 Homelab Emergency Powerdown" "[$TIMESTAMP] System shutdown initiated by NUT." 10 10038562
|
||||
logger -t nut-notify "System shutdown initiated."
|
||||
;;
|
||||
*)
|
||||
logger -t nut-notify "Unhandled UPS event: ${EVENT_TYPE}"
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
23
scripts/test_nut_alert.py
Normal file
23
scripts/test_nut_alert.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('14.4.4.44', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
payload = json.dumps({
|
||||
"title": "NUT Master Notification Pipeline Verified",
|
||||
"message": "Direct alert link verified: 14.4.4.44 -> Gotify VIP (http://140.44.4.70:8090) -> Discord #ups-power channel.",
|
||||
"priority": 5
|
||||
})
|
||||
|
||||
curl_cmd = f"curl -s -X POST \"http://140.44.4.70:8090/message?token=A4uN4fEmErGeNcYtOkEn\" -H \"Content-Type: application/json\" -d '{payload}'"
|
||||
stdin, stdout, stderr = ssh.exec_command(curl_cmd)
|
||||
res = stdout.read().decode()
|
||||
print("GOTIFY RESPONSE:", res)
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user