Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pillar-4): complete dual-npm keepalived HA cluster on VIP 140.44.4.70

Browse Source
This commit is contained in:
Snarf
2026-11-03 01:05:05 -05:00
parent 9391b3f2a1
commit 5cb00c04a9
19 changed files with 521 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/check_host_ports.py Normal file
View File

@@ -0,0 +1,21 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
for ip, node in [('140.44.4.71', 'snshpd4lnxswm01'), ('140.44.4.72', 'snshpd4lnxswm02')]:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print(f"=== {node} ({ip}) ===")
stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:81 |:443 " || true')
print("SS:\n", stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('sudo docker ps -a --filter "name=nginx" --format "{{.ID}} {{.Status}} {{.Names}}"')
print("Containers:\n", stdout.read().decode())
ssh.close()
if __name__ == '__main__':
main()

35
scripts/check_keepalived_all.py Normal file
View File

@@ -0,0 +1,35 @@
import paramiko
NODES = [
("swm01", "140.44.4.71"),
("swm02", "140.44.4.72"),
("swm03", "140.44.4.73"),
("swm04", "140.44.4.74"),
("swm05", "140.44.4.75"),
]
def main():
for name, ip in NODES:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
stdin, stdout, stderr = ssh.exec_command("systemctl is-active keepalived 2>/dev/null || echo inactive")
status = stdout.read().decode().strip()
stdin, stdout, stderr = ssh.exec_command("ip -4 addr show | grep '140.44.4.70' || true")
vip = stdout.read().decode().strip()
stdin, stdout, stderr = ssh.exec_command("grep -E 'state|priority' /etc/keepalived/keepalived.conf 2>/dev/null || true")
conf = stdout.read().decode().strip()
print(f"[{name} - {ip}] Keepalived: {status} | VIP: {'YES' if '140.44.4.70' in vip else 'NO'}")
if conf:
print(f" Config: {conf.splitlines()}")
ssh.close()
except Exception as e:
print(f"[{name} - {ip}] Error:", e)
if __name__ == '__main__':
main()

27
scripts/check_npm_status.py Normal file
View File

@@ -0,0 +1,27 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
for ip in ['140.44.4.71', '140.44.4.72']:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:443 |:81 " || true')
print(f"=== LISTENING PORTS ON {ip} ===")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" --format "{{.ID}} {{.Status}}"')
cid = stdout.read().decode().strip()
print(f"Container on {ip}: {cid}")
if cid:
cid_clean = cid.split()[0]
stdin, stdout, stderr = ssh.exec_command(f'sudo docker logs {cid_clean} --tail 15')
print(f"Logs on {ip}:\n{stdout.read().decode()}")
ssh.close()
if __name__ == '__main__':
main()

16
scripts/check_npm_volumes.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
def main():
for ip in ['140.44.4.71', '140.44.4.72']:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command("sudo docker volume inspect nginx-proxy-manager_nginx-data --format '{{json .Options}}'")
print(f"Volume data on {ip}: {stdout.read().decode().strip()}")
stdin, stdout, stderr = ssh.exec_command("sudo docker volume inspect nginx-proxy-manager_nginx-letsencrypt --format '{{json .Options}}'")
print(f"Volume letsencrypt on {ip}: {stdout.read().decode().strip()}")
ssh.close()
if __name__ == '__main__':
main()

16
scripts/check_port_forwards.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect('14.44.4.1', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
stdin, stdout, stderr = ssh.exec_command('iptables -t nat -S | grep -E "80|443"')
print("IPTABLES NAT:\n", stdout.read().decode())
ssh.close()
except Exception as e:
print("SSH to UDM Pro error:", e)
if __name__ == '__main__':
main()

15
scripts/check_swarm_ports.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
def main():
for ip in ['140.44.4.71', '140.44.4.72']:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:443 |:81 " || true')
print(f"=== PORTS ON {ip} ===")
print(stdout.read().decode())
ssh.close()
if __name__ == '__main__':
main()

23
scripts/check_swarm_service_status.py Normal file
View File

@@ -0,0 +1,23 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== DOCKER SERVICE LS ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker service ls')
print(stdout.read().decode())
print("=== DOCKER SERVICE PS ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app')
print(stdout.read().decode())
ssh.close()
if __name__ == '__main__':
main()

60
scripts/deploy_dual_npm.py Normal file
View File

@@ -0,0 +1,60 @@
import paramiko
import time
import urllib.request
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== STEP 1: Updating nginx-proxy-manager_app to Dual-Node Swarm Placement ===")
cmd = (
"sudo docker service update "
"--constraint-rm 'node.hostname == snshpd4lnxswm04' "
"--constraint-add 'node.labels.npm == true' "
"--replicas 2 "
"--update-order start-first "
"nginx-proxy-manager_app"
)
stdin, stdout, stderr = ssh.exec_command(cmd)
print("Service update output:\n", stdout.read().decode())
print("Service update stderr:\n", stderr.read().decode())
print("=== STEP 2: Waiting for tasks to stabilize ===")
time.sleep(10)
stdin, stdout, stderr = ssh.exec_command("sudo docker service ps nginx-proxy-manager_app --no-trunc")
print("Tasks:\n", stdout.read().decode())
ssh.close()
print("=== STEP 3: Verifying HTTP / HTTPS endpoints ===")
for ip in ['140.44.4.70', '140.44.4.71', '140.44.4.72']:
# Port 80
try:
req = urllib.request.Request(f"http://{ip}", headers={"Host": "test.snarfnet.net"})
with urllib.request.urlopen(req, timeout=4) as r:
print(f"[{ip}:80] HTTP {r.status}")
except urllib.error.HTTPError as e:
print(f"[{ip}:80] HTTP {e.code} (NPM responding)")
except Exception as e:
print(f"[{ip}:80] Error: {e}")
# Port 81 (Admin UI)
try:
req = urllib.request.Request(f"http://{ip}:81")
with urllib.request.urlopen(req, timeout=4) as r:
print(f"[{ip}:81] HTTP {r.status} (Admin UI responding)")
except urllib.error.HTTPError as e:
print(f"[{ip}:81] HTTP {e.code}")
except Exception as e:
print(f"[{ip}:81] Error: {e}")
if __name__ == '__main__':
main()

25
scripts/inspect_container_nginx.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" -q')
cid = stdout.read().decode().strip()
print("Container ID:", cid)
if cid:
stdin, stdout, stderr = ssh.exec_command(f'sudo docker exec {cid} nginx -t')
print("NGINX -t STDOUT:\n", stdout.read().decode())
print("NGINX -t STDERR:\n", stderr.read().decode())
stdin, stdout, stderr = ssh.exec_command(f'sudo docker exec {cid} curl -I http://127.0.0.1:80')
print("CURL 80 INSIDE:\n", stdout.read().decode())
print("CURL 80 STDERR:\n", stderr.read().decode())
ssh.close()
if __name__ == '__main__':
main()

25
scripts/inspect_keepalived_conf.py Normal file
View File

@@ -0,0 +1,25 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== /etc/keepalived/keepalived.conf ===")
stdin, stdout, stderr = ssh.exec_command('sudo cat /etc/keepalived/keepalived.conf')
print(stdout.read().decode())
print("=== /etc/keepalived/scripts/ ===")
stdin, stdout, stderr = ssh.exec_command('sudo ls -la /etc/keepalived/scripts/ || true')
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('sudo cat /etc/keepalived/scripts/* || true')
print(stdout.read().decode())
ssh.close()
if __name__ == '__main__':
main()

34
scripts/inspect_npm_72.py Normal file
View File

@@ -0,0 +1,34 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== DOCKER EXEC PS INSIDE f026d1c24ef6 ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker exec f026d1c24ef6 ps aux')
print(stdout.read().decode())
print(stderr.read().decode())
print("=== REDIRECTION_HOST LS ===")
stdin, stdout, stderr = ssh.exec_command('sudo ls -la /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/redirection_host || true')
print(stdout.read().decode())
print(stderr.read().decode())
print("=== RECENT DOCKER LOGS WITH TIMESTAMPS ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker logs f026d1c24ef6 --tail 30')
print(stdout.read().decode())
print("=== LOCAL CURL TEST ON 140.44.4.72 ===")
stdin, stdout, stderr = ssh.exec_command('curl -I http://127.0.0.1:80 ; curl -I http://127.0.0.1:81')
print(stdout.read().decode())
print(stderr.read().decode())
ssh.close()
if __name__ == '__main__':
main()

25
scripts/inspect_swarm_nodes_labels.py Normal file
View File

@@ -0,0 +1,25 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('sudo docker node ls --format "{{.Hostname}}"')
nodes = stdout.read().decode().split()
for n in nodes:
stdin, stdout, stderr = ssh.exec_command(f'sudo docker node inspect {n} --format "{{{{.Description.Hostname}}}}: {{{{.Spec.Labels}}}}"')
print(stdout.read().decode().strip())
print("\n=== DOCKER SERVICE PS ALL ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app --no-trunc')
print(stdout.read().decode())
ssh.close()
if __name__ == '__main__':
main()

1
scripts/pin_spaceport_vips.py
View File

@@ -51,3 +51,4 @@ def main():
if __name__ == '__main__':
main()

30
scripts/rescale_npm.py Normal file
View File

@@ -0,0 +1,30 @@
import sys
import time
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== SCALING TO 1 ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker service scale nginx-proxy-manager_app=1')
print(stdout.read().decode())
time.sleep(3)
print("=== SCALING TO 2 ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker service scale nginx-proxy-manager_app=2')
print(stdout.read().decode())
time.sleep(5)
print("=== SERVICE PS ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app --filter desired-state=running')
print(stdout.read().decode())
ssh.close()
if __name__ == '__main__':
main()

24
scripts/setup_dual_npm_cluster.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import json
import time
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== STEP 1: Add node labels 'npm=true' to swm01 (.71) and swm02 (.72) ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker node update --label-add npm=true snshpd4lnxswm01 && sudo docker node update --label-add npm=true snshpd4lnxswm02')
print("Labels update:", stdout.read().decode().strip(), stderr.read().decode().strip())
# Check node labels
stdin, stdout, stderr = ssh.exec_command("sudo docker node inspect snshpd4lnxswm01 --format '{{json .Spec.Labels}}'")
print("Labels swm01:", stdout.read().decode().strip())
stdin, stdout, stderr = ssh.exec_command("sudo docker node inspect snshpd4lnxswm02 --format '{{json .Spec.Labels}}'")
print("Labels swm02:", stdout.read().decode().strip())
ssh.close()
if __name__ == '__main__':
main()

66
scripts/test_keepalived_failover.py Normal file
View File

@@ -0,0 +1,66 @@
import sys
import time
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def check_vip(ssh):
stdin, stdout, stderr = ssh.exec_command("ip -4 addr show dev eth0 | grep '140.44.4.70' || true")
return bool(stdout.read().decode().strip())
def curl_vip(ssh):
stdin, stdout, stderr = ssh.exec_command("curl -s -o /dev/null -w '%{http_code}' -m 2 http://140.44.4.70:80 || true")
return stdout.read().decode().strip()
def main():
ssh1 = paramiko.SSHClient()
ssh1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh1.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
ssh2 = paramiko.SSHClient()
ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh2.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== INITIAL STATE ===")
v1 = check_vip(ssh1)
v2 = check_vip(ssh2)
code = curl_vip(ssh1)
print(f"swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
print("\n=== TRIGGERING FAILOVER: STOPPING KEEPALIVED ON swm01 ===")
ssh1.exec_command("sudo systemctl stop keepalived")
# Wait up to 5 seconds for swm02 to claim VIP
claimed = False
for i in range(5):
time.sleep(1)
if check_vip(ssh2):
claimed = True
break
v1 = check_vip(ssh1)
v2 = check_vip(ssh2)
code = curl_vip(ssh2)
print(f"Failover result -> swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
print("\n=== RESTORING swm01: STARTING KEEPALIVED ON swm01 ===")
ssh1.exec_command("sudo systemctl start keepalived")
# Wait up to 5 seconds for swm01 to reclaim VIP
reclaimed = False
for i in range(5):
time.sleep(1)
if check_vip(ssh1):
reclaimed = True
break
v1 = check_vip(ssh1)
v2 = check_vip(ssh2)
code = curl_vip(ssh1)
print(f"Fallback result -> swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
ssh1.close()
ssh2.close()
if __name__ == '__main__':
main()

26
scripts/test_npm_both_nodes.py Normal file
View File

@@ -0,0 +1,26 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
def main():
for ip, node in [('140.44.4.71', 'snshpd4lnxswm01'), ('140.44.4.72', 'snshpd4lnxswm02')]:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print(f"=== TESTING {node} ({ip}) ===")
stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" --format "{{.ID}} {{.Status}} {{.Names}}"')
print("Containers:", stdout.read().decode().strip())
stdin, stdout, stderr = ssh.exec_command('curl -I http://127.0.0.1:80 ; curl -I http://127.0.0.1:81')
print("Curls:\n", stdout.read().decode())
err = stderr.read().decode()
if err:
print("Errors:\n", err)
ssh.close()
if __name__ == '__main__':
main()

49
scripts/update_keepalived_healthcheck.py Normal file
View File

@@ -0,0 +1,49 @@
import sys
import paramiko
sys.stdout.reconfigure(encoding='utf-8')
NODES = [
("swm01", "140.44.4.71"),
("swm02", "140.44.4.72"),
("swm03", "140.44.4.73"),
("swm04", "140.44.4.74"),
("swm05", "140.44.4.75"),
]
SCRIPT_CONTENT = """#!/bin/bash
# Check docker daemon health
systemctl is-active --quiet docker || exit 1
# If NPM container is running on this node, verify HTTP responsiveness
if docker ps --format '{{.Names}}' 2>/dev/null | grep -q "nginx-proxy-manager_app"; then
curl -sf -m 2 http://127.0.0.1:80 >/dev/null 2>&1 || curl -sf -m 2 http://127.0.0.1:81 >/dev/null 2>&1 || exit 1
fi
exit 0
"""
def main():
for name, ip in NODES:
print(f"=== Updating {name} ({ip}) ===")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
# Write to temp file then sudo mv
sftp = ssh.open_sftp()
with sftp.open('/tmp/check_docker.sh', 'w') as f:
f.write(SCRIPT_CONTENT.replace('\r\n', '\n'))
sftp.close()
cmd = "sudo mv /tmp/check_docker.sh /etc/keepalived/scripts/check_docker.sh && sudo chmod +x /etc/keepalived/scripts/check_docker.sh && sudo /etc/keepalived/scripts/check_docker.sh && echo 'TEST: OK'"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode().strip())
err = stderr.read().decode().strip()
if err:
print("ERR:", err)
ssh.close()
if __name__ == '__main__':
main()