feat: add homelab topology documentation, infrastructure improvement plans, and management scripts
This commit is contained in:
BIN
scripts/__pycache__/harden_online_rpi_boot.cpython-312.pyc
Normal file
BIN
scripts/__pycache__/harden_online_rpi_boot.cpython-312.pyc
Normal file
Binary file not shown.
266
scripts/gotify_discord_bridge.py
Normal file
266
scripts/gotify_discord_bridge.py
Normal file
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Gotify <-> Home Assistant <-> Discord Multi-Channel Bridge for Snarfnet Homelab.
|
||||
Zero external dependencies (Python 3 standard library only).
|
||||
1. Watches Home Assistant (http://14.4.4.20:8123) logbook.log broadcast events
|
||||
and publishes new notifications to Gotify ('Home Assistant Smart Home', appid=3).
|
||||
2. Polls Gotify (http://gotify:80) and routes all 5 broadcast applications to their
|
||||
dedicated Discord channel webhooks (#ups-power, #infra-alerts, #home-assistant)
|
||||
with priority-colored rich embeds, persistent cursor tracking, and HTTP 429 backoff.
|
||||
"""
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
GOTIFY_URL = os.environ.get("GOTIFY_URL", "http://gotify:80").rstrip("/")
|
||||
GOTIFY_CLIENT_TOKEN = os.environ.get("GOTIFY_CLIENT_TOKEN", "C1dIsCoRdBrIdGeClIeN")
|
||||
GOTIFY_HA_APP_TOKEN = os.environ.get("GOTIFY_HA_APP_TOKEN", "A3hOmEaSsIsTaNtToKeN")
|
||||
STATE_FILE = os.environ.get("STATE_FILE", "/app/data/discord_bridge_last_id.txt")
|
||||
HA_STATE_FILE = os.environ.get("HA_STATE_FILE", "/app/data/ha_last_logbook_when.txt")
|
||||
POLL_INTERVAL = float(os.environ.get("POLL_INTERVAL", "2.0"))
|
||||
|
||||
HA_URL = os.environ.get("HA_URL", "http://14.4.4.20:8123").rstrip("/")
|
||||
HA_TOKEN = os.environ.get(
|
||||
"HA_TOKEN",
|
||||
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkZWNlMWI5ZjJhZDQ0MTRmODU4NGYzN2I1YjgwZDQxMiIsImlhdCI6MTc5MzQ1MjQ2MywiZXhwIjoyMTA4ODEyNDYzfQ.JznpOqsCSnHz6aTVa8K9pQCz1816DC8SrmS1xn1Sm2k",
|
||||
)
|
||||
|
||||
WEBHOOK_UPS = os.environ.get(
|
||||
"WEBHOOK_UPS",
|
||||
"https://discord.com/api/webhooks/1566076118535114813/2e_fCJeHb-mF8VwerzOORGRa-QUDFuwN6R2Gfz-MGUiIOudA0PiXGZSyd19PNFG-vR7D",
|
||||
)
|
||||
WEBHOOK_INFRA = os.environ.get(
|
||||
"WEBHOOK_INFRA",
|
||||
"https://discord.com/api/webhooks/1566076777963593772/wKN30N0ZXVJKGP-prZ1_P9cp_u7Oakmh8R4bVh0qr8RFpyeGcJZrHfPQBJxtfzQqb4dn",
|
||||
)
|
||||
WEBHOOK_HA = os.environ.get(
|
||||
"WEBHOOK_HA",
|
||||
"https://discord.com/api/webhooks/1566077255837294684/4x_sTqDeEHfIeGccEutOSvPKAnCV6Fyr7uAuECOt8M8QZcOvL2XZ0GVATW7EmcSyUW33",
|
||||
)
|
||||
|
||||
# Map Gotify Application ID -> (Channel Name, Discord Webhook URL)
|
||||
APP_WEBHOOK_MAP = {
|
||||
1: ("ups-power", WEBHOOK_UPS),
|
||||
2: ("infra-alerts", WEBHOOK_INFRA),
|
||||
3: ("home-assistant", WEBHOOK_HA),
|
||||
4: ("infra-alerts", WEBHOOK_INFRA),
|
||||
5: ("infra-alerts", WEBHOOK_INFRA),
|
||||
}
|
||||
|
||||
APP_NAMES = {
|
||||
1: "UPS & Power Resilience",
|
||||
2: "Zabbix Infrastructure Alerts",
|
||||
3: "Home Assistant Smart Home",
|
||||
4: "AWX & GitOps Automation",
|
||||
5: "Backups & Storage Health",
|
||||
}
|
||||
|
||||
ssl_ctx = ssl.create_default_context()
|
||||
|
||||
|
||||
def priority_color(priority: int) -> int:
|
||||
if priority >= 8:
|
||||
return 15158332 # Crimson Red
|
||||
if priority >= 5:
|
||||
return 15965202 # Amber Orange
|
||||
return 3447003 # Cobalt Blue
|
||||
|
||||
|
||||
def load_last_id():
|
||||
try:
|
||||
if os.path.exists(STATE_FILE):
|
||||
with open(STATE_FILE, "r", encoding="utf-8") as f:
|
||||
return int(f.read().strip())
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def save_last_id(msg_id: int):
|
||||
try:
|
||||
with open(STATE_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(msg_id))
|
||||
except Exception as e:
|
||||
print(f"[WARN] Could not write state file {STATE_FILE}: {e}", flush=True)
|
||||
|
||||
|
||||
def load_ha_when() -> str:
|
||||
try:
|
||||
if os.path.exists(HA_STATE_FILE):
|
||||
with open(HA_STATE_FILE, "r", encoding="utf-8") as f:
|
||||
val = f.read().strip()
|
||||
if val:
|
||||
return val
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def save_ha_when(when_str: str):
|
||||
try:
|
||||
with open(HA_STATE_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(when_str)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Could not write HA state file {HA_STATE_FILE}: {e}", flush=True)
|
||||
|
||||
|
||||
def render_ha_template(tpl: str) -> str:
|
||||
if "{{" not in tpl:
|
||||
return tpl
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{HA_URL}/api/template",
|
||||
data=json.dumps({"template": tpl}).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {HA_TOKEN}", "Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=6) as r:
|
||||
return r.read().decode("utf-8").strip()
|
||||
except Exception:
|
||||
return tpl
|
||||
|
||||
|
||||
def poll_home_assistant(ha_last_when: str) -> str:
|
||||
if not HA_TOKEN:
|
||||
return ha_last_when
|
||||
|
||||
ts_query = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=2)).isoformat()
|
||||
req = urllib.request.Request(
|
||||
f"{HA_URL}/api/logbook/{ts_query}?entity=script.snarfnet_broadcast",
|
||||
headers={"Authorization": f"Bearer {HA_TOKEN}", "Accept": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
entries = json.loads(r.read().decode("utf-8"))
|
||||
|
||||
log_entries = [
|
||||
e for e in entries
|
||||
if e.get("entity_id") == "script.snarfnet_broadcast"
|
||||
and e.get("message")
|
||||
and e.get("message") != "started"
|
||||
and e.get("when")
|
||||
]
|
||||
|
||||
if ha_last_when is None:
|
||||
ha_last_when = max((e["when"] for e in log_entries), default="")
|
||||
save_ha_when(ha_last_when)
|
||||
print(f"[INIT-HA] Initialized Home Assistant logbook cursor at '{ha_last_when}'", flush=True)
|
||||
return ha_last_when
|
||||
|
||||
for e in sorted(log_entries, key=lambda x: x["when"]):
|
||||
when_str = e["when"]
|
||||
if when_str > ha_last_when:
|
||||
title = render_ha_template(e.get("name") or "Home Assistant Alert")
|
||||
message = render_ha_template(e.get("message") or "(no message)")
|
||||
prio = 8 if any(w in title.lower() or w in message.lower() for w in ["critical", "failed", "stalled", "battery", "outage", "high", "warning", "alarm"]) else 5
|
||||
t_str = title if title.startswith(("🏠", "🖨️", "✅", "⚠️", "🚨", "⚡", "🌡️")) else f"🏠 {title}"
|
||||
g_req = urllib.request.Request(
|
||||
f"{GOTIFY_URL}/message?token={GOTIFY_HA_APP_TOKEN}",
|
||||
data=json.dumps({"title": t_str, "message": message, "priority": prio}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(g_req, timeout=6) as gr:
|
||||
gid = json.loads(gr.read().decode("utf-8")).get("id")
|
||||
print(f"[HA->GOTIFY] Published '{title}' as Gotify msg #{gid}", flush=True)
|
||||
ha_last_when = when_str
|
||||
save_ha_when(ha_last_when)
|
||||
|
||||
return ha_last_when
|
||||
|
||||
|
||||
def fetch_messages():
|
||||
req = urllib.request.Request(
|
||||
f"{GOTIFY_URL}/message?limit=25",
|
||||
headers={"X-Gotify-Key": GOTIFY_CLIENT_TOKEN, "Accept": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
data = json.loads(r.read().decode("utf-8"))
|
||||
return data.get("messages", [])
|
||||
|
||||
|
||||
def send_discord(msg: dict) -> bool:
|
||||
app_id = msg.get("appid", 0)
|
||||
chan_label, webhook_url = APP_WEBHOOK_MAP.get(app_id, ("infra-alerts", WEBHOOK_INFRA))
|
||||
app_name = APP_NAMES.get(app_id, f"Gotify App #{app_id}")
|
||||
priority = int(msg.get("priority", 5))
|
||||
title = msg.get("title") or app_name
|
||||
body = msg.get("message") or "(no content)"
|
||||
date_str = msg.get("date")
|
||||
|
||||
embed = {
|
||||
"title": title[:256],
|
||||
"description": body[:4000],
|
||||
"color": priority_color(priority),
|
||||
"footer": {"text": f"Snarfnet Gotify • {app_name} • Priority {priority}"},
|
||||
}
|
||||
if date_str:
|
||||
embed["timestamp"] = date_str
|
||||
|
||||
payload = {
|
||||
"username": f"Snarfnet • {app_name}",
|
||||
"embeds": [embed],
|
||||
}
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
webhook_url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "SnarfNet-GotifyDiscordBridge/1.0",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10, context=ssl_ctx) as r:
|
||||
if r.status in (200, 204):
|
||||
print(f"[OK] Forwarded msg #{msg['id']} ({app_name}) -> #{chan_label}", flush=True)
|
||||
return True
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
retry_after = float(e.headers.get("Retry-After", "2.5"))
|
||||
print(f"[RATE-LIMIT] HTTP 429 on #{chan_label}, sleeping {retry_after}s...", flush=True)
|
||||
time.sleep(retry_after)
|
||||
continue
|
||||
print(f"[ERROR] Discord HTTP {e.code} on msg #{msg['id']}: {e.read().decode('utf-8', errors='ignore')}", flush=True)
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Discord exception on msg #{msg['id']} (attempt {attempt + 1}): {e}", flush=True)
|
||||
time.sleep(1.5)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
print(f"[START] Snarfnet Gotify <-> HA <-> Discord Bridge polling {GOTIFY_URL} & {HA_URL}", flush=True)
|
||||
last_id = load_last_id()
|
||||
ha_last_when = load_ha_when()
|
||||
|
||||
while True:
|
||||
try:
|
||||
ha_last_when = poll_home_assistant(ha_last_when)
|
||||
except Exception as e:
|
||||
print(f"[WARN] HA poll error: {e}", flush=True)
|
||||
|
||||
try:
|
||||
messages = fetch_messages()
|
||||
if last_id is None:
|
||||
last_id = max((m["id"] for m in messages), default=0)
|
||||
save_last_id(last_id)
|
||||
print(f"[INIT] Initialized cursor at message ID {last_id}", flush=True)
|
||||
else:
|
||||
new_msgs = sorted((m for m in messages if m["id"] > last_id), key=lambda x: x["id"])
|
||||
for m in new_msgs:
|
||||
send_discord(m)
|
||||
last_id = m["id"]
|
||||
save_last_id(last_id)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Gotify poll error: {e}", flush=True)
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
172
scripts/harden_online_rpi_boot.py
Normal file
172
scripts/harden_online_rpi_boot.py
Normal file
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Proactively harden boot & filesystem configuration across all 24 online Snarfnet RPi/Linux nodes:
|
||||
1. Ensures 'fsck.repair=yes' and 'rootwait' are set in /boot/firmware/cmdline.txt (and /boot/cmdline.txt).
|
||||
2. Sets ext4 root superblock error behavior to 'continue' (tune2fs -e continue).
|
||||
3. Adds 'nofail,x-systemd.automount' to GlusterFS and NFS network mounts in /etc/fstab.
|
||||
4. Adds 'nofail' to /boot/firmware (/boot) vfat mount in /etc/fstab so a dirty FAT bit never blocks SSH boot.
|
||||
5. Reloads systemd and recovers failed network mounts on Shipyard Swarm nodes.
|
||||
Uses pure SSH stdin (no SFTP required) for full compatibility with DietPi Dropbear SSH.
|
||||
"""
|
||||
import base64
|
||||
import sys
|
||||
import time
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
ONLINE_NODES = [
|
||||
("aerodrome-ctl01", "140.44.4.11", "root", "Katzir476!"),
|
||||
("aerodrome-ctl02", "140.44.4.12", "root", "Katzir476!"),
|
||||
("aerodrome-ctl03", "140.44.4.13", "root", "Katzir476!"),
|
||||
("aerodrome-wrk01", "140.44.4.21", "root", "Katzir476!"),
|
||||
("aerodrome-wrk02", "140.44.4.22", "root", "Katzir476!"),
|
||||
("aerodrome-wrk03", "140.44.4.23", "root", "Katzir476!"),
|
||||
("aerodrome-wrk04", "140.44.4.24", "root", "Katzir476!"),
|
||||
("aerodrome-wrk05", "140.44.4.25", "root", "Katzir476!"),
|
||||
("aerodrome-wrk06", "140.44.4.26", "root", "Katzir476!"),
|
||||
("aerodrome-wrk07", "140.44.4.27", "root", "Katzir476!"),
|
||||
("spaceport-ctl01", "140.44.4.41", "root", "dietpi"),
|
||||
("spaceport-ctl02", "140.44.4.42", "root", "dietpi"),
|
||||
("spaceport-ctl03", "140.44.4.43", "root", "dietpi"),
|
||||
("spaceport-ctl04", "140.44.4.44", "root", "dietpi"),
|
||||
("spaceport-wrk01", "140.44.4.51", "root", "dietpi"),
|
||||
("spaceport-wrk02", "140.44.4.52", "root", "dietpi"),
|
||||
("spaceport-wrk03", "140.44.4.53", "root", "dietpi"),
|
||||
("spaceport-wrk04", "140.44.4.54", "root", "dietpi"),
|
||||
("spaceport-wrk05", "140.44.4.55", "root", "Katzir476!"),
|
||||
("spaceport-wrk06", "140.44.4.56", "root", "dietpi"),
|
||||
("shipyard-swm01", "140.44.4.71", "snarf", "Katzir476!"),
|
||||
("shipyard-swm02", "140.44.4.72", "snarf", "Katzir476!"),
|
||||
("shipyard-swm03", "140.44.4.73", "snarf", "Katzir476!"),
|
||||
("shipyard-swm04", "140.44.4.74", "snarf", "Katzir476!"),
|
||||
("shipyard-swm05", "140.44.4.75", "snarf", "Katzir476!"),
|
||||
("ai-ops01", "140.44.4.7", "snarf", "Katzir476!"),
|
||||
("ai-web01", "140.44.4.8", "snarf", "Katzir476!"),
|
||||
("nutserver", "14.4.4.44", "snarf", "Katzir476!"),
|
||||
]
|
||||
|
||||
HARDEN_SH = r"""
|
||||
set -e
|
||||
HOST=$(hostname)
|
||||
|
||||
# 1. Tune root ext4 filesystem error behavior to 'continue'
|
||||
ROOT_DEV=$(findmnt -n -o SOURCE /)
|
||||
if [[ "$ROOT_DEV" == /dev/* ]]; then
|
||||
tune2fs -e continue "$ROOT_DEV" >/dev/null 2>&1 || true
|
||||
ERR_BEHAV=$(tune2fs -l "$ROOT_DEV" 2>/dev/null | grep "Errors behavior" | awk -F: '{print $2}' | xargs)
|
||||
echo " [EXT4] $ROOT_DEV -> Errors behavior: $ERR_BEHAV"
|
||||
fi
|
||||
|
||||
# 2. Ensure fsck.repair=yes (and usb-storage.quirks=152d:0583:u if JMicron bridge present) in active cmdline.txt files
|
||||
HAS_JMICRON=0
|
||||
if lsusb 2>/dev/null | grep -qi "152d:0583"; then
|
||||
HAS_JMICRON=1
|
||||
fi
|
||||
for cmdfile in /boot/firmware/cmdline.txt /boot/cmdline.txt; do
|
||||
if [ -f "$cmdfile" ] && ! grep -q "DO NOT EDIT" "$cmdfile"; then
|
||||
if ! grep -q "fsck.repair=yes" "$cmdfile"; then
|
||||
cp "$cmdfile" "${cmdfile}.bak"
|
||||
sed -i 's/rootwait/fsck.repair=yes rootwait/g' "$cmdfile"
|
||||
echo " [CMDLINE] Added fsck.repair=yes to $cmdfile"
|
||||
else
|
||||
echo " [CMDLINE] Verified fsck.repair=yes already in $cmdfile"
|
||||
fi
|
||||
if [ "$HAS_JMICRON" -eq 1 ] && ! grep -q "usb-storage.quirks=152d:0583:u" "$cmdfile"; then
|
||||
cp "$cmdfile" "${cmdfile}.bak"
|
||||
sed -i 's/^/usb-storage.quirks=152d:0583:u /' "$cmdfile"
|
||||
echo " [CMDLINE] Added usb-storage.quirks=152d:0583:u to $cmdfile"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# 3. Harden /etc/fstab (add nofail,x-systemd.automount to glusterfs/nfs; add nofail to /boot vfat)
|
||||
cp /etc/fstab /etc/fstab.bak
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
python3 -c '
|
||||
with open("/etc/fstab", "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
new_lines = []
|
||||
changed = False
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
new_lines.append(line)
|
||||
continue
|
||||
parts = stripped.split()
|
||||
if len(parts) >= 4:
|
||||
mp, fstype, opts = parts[1], parts[2], parts[3]
|
||||
opt_list = opts.split(",")
|
||||
if fstype in ("glusterfs", "nfs", "nfs4", "cifs"):
|
||||
for req_opt in ("_netdev", "nofail", "x-systemd.automount"):
|
||||
if req_opt not in opt_list:
|
||||
opt_list.append(req_opt)
|
||||
changed = True
|
||||
elif mp in ("/boot", "/boot/firmware") and fstype == "vfat":
|
||||
if "nofail" not in opt_list:
|
||||
opt_list.append("nofail")
|
||||
changed = True
|
||||
parts[3] = ",".join(opt_list)
|
||||
new_lines.append("\t".join(parts) + "\n")
|
||||
else:
|
||||
new_lines.append(line)
|
||||
|
||||
if changed:
|
||||
with open("/etc/fstab", "w") as f:
|
||||
f.writelines(new_lines)
|
||||
print(" [FSTAB] Updated /etc/fstab with nofail / x-systemd.automount")
|
||||
else:
|
||||
print(" [FSTAB] Verified /etc/fstab already has nofail options")
|
||||
'
|
||||
else
|
||||
if grep -E "/boot(/firmware)?[[:space:]]+vfat" /etc/fstab | grep -vq "nofail"; then
|
||||
sed -i -E 's|(/boot(/firmware)?[[:space:]]+vfat[[:space:]]+[^[:space:]]+)|\1,nofail|g' /etc/fstab
|
||||
echo " [FSTAB] Added nofail to /boot vfat in /etc/fstab (via sed)"
|
||||
else
|
||||
echo " [FSTAB] Verified /etc/fstab already has nofail on /boot"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Reload systemd, reset failed units, and ensure mounts are active
|
||||
systemctl daemon-reload
|
||||
systemctl reset-failed 2>/dev/null || true
|
||||
if grep -q "/mnt/nfs_shares" /etc/fstab; then
|
||||
mount /mnt/nfs_shares 2>/dev/null || true
|
||||
mount /mnt/swarm_shared 2>/dev/null || true
|
||||
fi
|
||||
FAILED_CNT=$(systemctl --failed --no-legend | wc -l)
|
||||
echo " [SYSTEMD] Failed units count: $FAILED_CNT"
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
b64_script = base64.b64encode(HARDEN_SH.encode("utf-8")).decode("ascii")
|
||||
for name, ip, user, pw in ONLINE_NODES:
|
||||
print(f"=== Hardening {name} ({ip}) ===")
|
||||
for attempt in range(2):
|
||||
try:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username=user, password=pw, timeout=10, banner_timeout=15, auth_timeout=15, look_for_keys=False, allow_agent=False)
|
||||
if user == "root":
|
||||
cmd = f"echo '{b64_script}' | base64 -d | bash"
|
||||
else:
|
||||
cmd = f"echo '{pw}' | sudo -S -p '' bash -c \"echo '{b64_script}' | base64 -d | bash\""
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=25)
|
||||
print(stdout.read().decode("utf-8", errors="ignore").strip())
|
||||
err = stderr.read().decode("utf-8", errors="ignore").strip()
|
||||
if err:
|
||||
print(f" [STDERR] {err}")
|
||||
ssh.close()
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 0:
|
||||
time.sleep(1.5)
|
||||
else:
|
||||
print(f" [ERROR] {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -11,31 +11,41 @@ TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
GOTIFY_INTERNAL_URL="http://140.44.4.70:8090/message"
|
||||
GOTIFY_APP_TOKEN="${GOTIFY_APP_TOKEN:-A4uN4fEmErGeNcYtOkEn}"
|
||||
|
||||
# Direct Discord fallback webhook (used if Shipyard/Gotify is already down during Tier 4-6 shutdown)
|
||||
DISCORD_UPS_WEBHOOK="${DISCORD_UPS_WEBHOOK:-https://discord.com/api/webhooks/1566076118535114813/2e_fCJeHb-mF8VwerzOORGRa-QUDFuwN6R2Gfz-MGUiIOudA0PiXGZSyd19PNFG-vR7D}"
|
||||
|
||||
send_gotify() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
local priority="$3"
|
||||
local color="${4:-16711680}"
|
||||
|
||||
curl -s -m 5 -X POST "${GOTIFY_INTERNAL_URL}?token=${GOTIFY_APP_TOKEN}" \
|
||||
# Primary: Send to Gotify (which pushes to mobile devices + forwards to Discord ups-power webhook)
|
||||
if ! curl -sf -m 5 -X POST "${GOTIFY_INTERNAL_URL}?token=${GOTIFY_APP_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"${title}\", \"message\": \"${message}\", \"priority\": ${priority}}" >/dev/null 2>&1 || true
|
||||
-d "{\"title\": \"${title}\", \"message\": \"${message}\", \"priority\": ${priority}}" >/dev/null 2>&1; then
|
||||
# Fallback: If Gotify/Shipyard is already powered down (Tier 4+), send directly to Discord via UDM Pro (Tier 6)
|
||||
curl -sf -m 5 -X POST "${DISCORD_UPS_WEBHOOK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\": \"SnarfNet NUT Master (Direct Fallback)\", \"embeds\": [{\"title\": \"${title}\", \"description\": \"${message}\", \"color\": ${color}}]}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
case "$EVENT_TYPE" in
|
||||
ONBATT)
|
||||
send_gotify "⚡ Grid Power Lost" "[$TIMESTAMP] Homelab is running on UPS battery (${UPS_NAME}). Staged shutdown countdown started." 8
|
||||
send_gotify "⚡ Grid Power Lost" "[$TIMESTAMP] Homelab is running on UPS battery (${UPS_NAME}). Staged shutdown countdown started." 8 16753920
|
||||
logger -t nut-notify "Grid power failure detected - running on UPS battery."
|
||||
;;
|
||||
ONLINE)
|
||||
send_gotify "✅ Grid Power Restored" "[$TIMESTAMP] Utility AC power restored to ${UPS_NAME}. Battery recharging." 5
|
||||
send_gotify "✅ Grid Power Restored" "[$TIMESTAMP] Utility AC power restored to ${UPS_NAME}. Battery recharging." 5 3066993
|
||||
logger -t nut-notify "AC utility power restored."
|
||||
;;
|
||||
LOWBATT)
|
||||
send_gotify "🚨 UPS Battery Critical" "[$TIMESTAMP] Battery level critical on ${UPS_NAME}! Final Tier 4-6 shutdown initiated." 10
|
||||
send_gotify "🚨 UPS Battery Critical" "[$TIMESTAMP] Battery level critical on ${UPS_NAME}! Final Tier 4-6 shutdown initiated." 10 15158332
|
||||
logger -t nut-notify "CRITICAL: UPS battery low. Enforcing emergency shutdown."
|
||||
;;
|
||||
FSD)
|
||||
send_gotify "🛑 Forced Shutdown Engaged" "[$TIMESTAMP] NUT Master has commanded immediate power cutoff across all clusters." 10
|
||||
send_gotify "🛑 Forced Shutdown Engaged" "[$TIMESTAMP] NUT Master has commanded immediate power cutoff across all clusters." 10 10038562
|
||||
logger -t nut-notify "FSD signal received. Halting systems."
|
||||
;;
|
||||
*)
|
||||
|
||||
156
scripts/repair_offline_rpi_ssd.sh
Normal file
156
scripts/repair_offline_rpi_ssd.sh
Normal file
@@ -0,0 +1,156 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================
|
||||
# Snarfnet Raspberry Pi SSD Boot Diagnostic & Automated Repair Script
|
||||
# Run this script after booting an offline node (.21, .27, .73, .75) from a
|
||||
# rescue SD card with the node's target SSD attached.
|
||||
# ==============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Snarfnet RPi Offline Node SSD Diagnostic & Repair Tool"
|
||||
echo "=================================================================="
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "[ERROR] Please run as root (sudo bash $0)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 1. Identify which device is currently mounted as the Rescue OS root (/)
|
||||
RESCUE_ROOT_PART=$(findmnt -n -o SOURCE /)
|
||||
RESCUE_DISK=$(lsblk -no PKNAME "$RESCUE_ROOT_PART" 2>/dev/null | head -n1)
|
||||
echo "[INFO] Rescue OS is running from: $RESCUE_ROOT_PART (disk: /dev/$RESCUE_DISK)"
|
||||
|
||||
echo ""
|
||||
echo "--- Attached Block Devices ---"
|
||||
lsblk -o NAME,SIZE,TYPE,FSTYPE,LABEL,MOUNTPOINT,PARTUUID,MODEL,TRAN
|
||||
echo "------------------------------"
|
||||
|
||||
# 2. Auto-detect target SSD (prefer /dev/sda or /dev/nvme0n1 that is NOT the rescue disk)
|
||||
TARGET_DISK="${1:-}"
|
||||
if [ -z "$TARGET_DISK" ]; then
|
||||
for cand in /dev/sda /dev/nvme0n1 /dev/sdb; do
|
||||
if [ -b "$cand" ] && [ "$(basename "$cand")" != "$RESCUE_DISK" ]; then
|
||||
TARGET_DISK="$cand"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "$TARGET_DISK" ] || [ ! -b "$TARGET_DISK" ]; then
|
||||
echo "[WARN] Target SSD not detected yet!"
|
||||
echo " -> If this is a Shipyard Pi 5 (.73 or .75), plug in the USB SSD now and wait 5 seconds."
|
||||
echo " -> Usage: sudo bash $0 /dev/sda"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[INFO] Target drive selected for repair: $TARGET_DISK"
|
||||
|
||||
# Determine partition names (/dev/sda1 + /dev/sda2 vs /dev/nvme0n1p1 + /dev/nvme0n1p2)
|
||||
if [[ "$TARGET_DISK" == *"nvme"* ]] || [[ "$TARGET_DISK" == *"mmcblk"* ]]; then
|
||||
BOOT_PART="${TARGET_DISK}p1"
|
||||
ROOT_PART="${TARGET_DISK}p2"
|
||||
else
|
||||
BOOT_PART="${TARGET_DISK}1"
|
||||
ROOT_PART="${TARGET_DISK}2"
|
||||
fi
|
||||
|
||||
if [ ! -b "$BOOT_PART" ] || [ ! -b "$ROOT_PART" ]; then
|
||||
echo "[ERROR] Expected partitions $BOOT_PART and $ROOT_PART not found on $TARGET_DISK!"
|
||||
lsblk "$TARGET_DISK"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure target partitions are unmounted before running fsck
|
||||
umount "$BOOT_PART" 2>/dev/null || true
|
||||
umount "$ROOT_PART" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== STEP 1: Checking SMART / USB Bridge Health ==="
|
||||
lsusb || true
|
||||
dmesg | grep -iE "usb|uas|sda|nvme|I/O error|reset" | tail -n 15 || true
|
||||
|
||||
echo ""
|
||||
echo "=== STEP 2: Repairing Boot FAT32 Partition ($BOOT_PART) ==="
|
||||
fsck.vfat -a -v "$BOOT_PART" || true
|
||||
|
||||
echo ""
|
||||
echo "=== STEP 3: Repairing Root EXT4 Partition ($ROOT_PART) ==="
|
||||
echo "Before repair superblock state:"
|
||||
tune2fs -l "$ROOT_PART" | grep -E "Filesystem state|Mount count|Last mount time|Last write time|Errors behavior" || true
|
||||
|
||||
# Force full ext4 journal & inode/bitmap repair (-f -y)
|
||||
e2fsck -f -y -v "$ROOT_PART" || true
|
||||
|
||||
# Set ext4 error behavior to 'continue' so minor block errors don't force read-only remount on boot
|
||||
tune2fs -e continue "$ROOT_PART"
|
||||
|
||||
echo "After repair superblock state:"
|
||||
tune2fs -l "$ROOT_PART" | grep -E "Filesystem state|Errors behavior" || true
|
||||
|
||||
echo ""
|
||||
echo "=== STEP 4: Mounting Target OS & Auditing Boot Config ==="
|
||||
mkdir -p /mnt/recovery
|
||||
mount "$ROOT_PART" /mnt/recovery
|
||||
|
||||
if [ -d /mnt/recovery/boot/firmware ]; then
|
||||
BOOT_MNT="/mnt/recovery/boot/firmware"
|
||||
else
|
||||
BOOT_MNT="/mnt/recovery/boot"
|
||||
fi
|
||||
mount "$BOOT_PART" "$BOOT_MNT"
|
||||
|
||||
NODE_HOSTNAME=$(cat /mnt/recovery/etc/hostname 2>/dev/null || echo "unknown")
|
||||
echo "[INFO] Mounted Node Hostname: $NODE_HOSTNAME"
|
||||
|
||||
ACTUAL_BOOT_PARTUUID=$(blkid -s PARTUUID -o value "$BOOT_PART")
|
||||
ACTUAL_ROOT_PARTUUID=$(blkid -s PARTUUID -o value "$ROOT_PART")
|
||||
echo "[INFO] Actual PARTUUIDs -> Boot ($BOOT_PART): $ACTUAL_BOOT_PARTUUID | Root ($ROOT_PART): $ACTUAL_ROOT_PARTUUID"
|
||||
|
||||
echo ""
|
||||
echo "--- Current /etc/fstab on $NODE_HOSTNAME ---"
|
||||
cat /mnt/recovery/etc/fstab
|
||||
|
||||
# Backup fstab before hardening
|
||||
cp /mnt/recovery/etc/fstab /mnt/recovery/etc/fstab.bak.$(date +%s)
|
||||
|
||||
# Fix 1: Add 'nofail,x-systemd.automount' to GlusterFS and NFS network mounts so they NEVER hang boot
|
||||
if grep -qE "glusterfs|nfs" /mnt/recovery/etc/fstab; then
|
||||
sed -i -E 's/(glusterfs[[:space:]]+defaults,_netdev)/\1,nofail,x-systemd.automount/g' /mnt/recovery/etc/fstab
|
||||
sed -i -E 's/(nfs[[:space:]]+defaults,nfsvers=4,_netdev)/\1,nofail,x-systemd.automount/g' /mnt/recovery/etc/fstab
|
||||
echo "[FIXED] Added 'nofail,x-systemd.automount' to network mounts in /etc/fstab."
|
||||
fi
|
||||
|
||||
# Fix 2: Ensure cmdline.txt has 'fsck.repair=yes' and 'rootwait' (missing on DietPi nodes!)
|
||||
for cmdfile in "$BOOT_MNT/cmdline.txt" "/mnt/recovery/boot/cmdline.txt"; do
|
||||
if [ -f "$cmdfile" ] && ! grep -q "DO NOT EDIT" "$cmdfile"; then
|
||||
cp "$cmdfile" "${cmdfile}.bak.$(date +%s)"
|
||||
if ! grep -q "fsck.repair=yes" "$cmdfile"; then
|
||||
sed -i 's/rootwait/fsck.repair=yes rootwait/g' "$cmdfile"
|
||||
echo "[FIXED] Added 'fsck.repair=yes' to $cmdfile"
|
||||
fi
|
||||
echo "Updated $cmdfile:"
|
||||
cat "$cmdfile"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== STEP 5: Checking Previous Boot Logs on Target Drive ==="
|
||||
if [ -d /mnt/recovery/var/log/journal ]; then
|
||||
journalctl -D /mnt/recovery/var/log/journal -b -1 -p 0..3 --no-pager | tail -n 30 || true
|
||||
else
|
||||
echo "[INFO] Node uses RAMlog (/var/log on tmpfs); no persistent journal on disk."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== STEP 6: Clean Unmount ==="
|
||||
sync
|
||||
umount "$BOOT_MNT"
|
||||
umount /mnt/recovery
|
||||
sync
|
||||
|
||||
echo "=================================================================="
|
||||
echo " [SUCCESS] Repair completed for $NODE_HOSTNAME ($TARGET_DISK)!"
|
||||
echo " You can now run 'sudo poweroff', remove the rescue SD card,"
|
||||
echo " and power the node back on to boot normally from its SSD."
|
||||
echo "=================================================================="
|
||||
|
||||
93
scripts/repair_sd_card_windows.ps1
Normal file
93
scripts/repair_sd_card_windows.ps1
Normal file
@@ -0,0 +1,93 @@
|
||||
# Run this script in an Administrator PowerShell prompt to inspect the Linux ext4
|
||||
# superblock on Partition 2 and patch H:\cmdline.txt with 'fsck.repair=yes ro'.
|
||||
|
||||
$ErrorActionPreference = "Continue"
|
||||
|
||||
$part1 = Get-Partition -DriveLetter H -ErrorAction SilentlyContinue
|
||||
if (-not $part1) {
|
||||
Write-Error "Drive H: not found! Make sure the SD card / SSD is inserted and mounted at H:."
|
||||
exit 1
|
||||
}
|
||||
|
||||
$diskNum = $part1.DiskNumber
|
||||
$disk = Get-Disk -Number $diskNum
|
||||
$part2 = Get-Partition -DiskNumber $diskNum -PartitionNumber 2 -ErrorAction SilentlyContinue
|
||||
$physDrive = "\\.\PhysicalDrive$diskNum"
|
||||
|
||||
Write-Host "=== 1. Disk & Partition Summary ===" -ForegroundColor Cyan
|
||||
Write-Host ("Disk {0}: {1} ({2}, Bus={3}, Size={4:N2} GB)" -f $diskNum, $disk.FriendlyName, $disk.HealthStatus, $disk.BusType, ($disk.Size / 1GB))
|
||||
if ($part2) {
|
||||
Write-Host ("Partition 2 (Linux ext4): Offset={0}, Size={1:N2} GB" -f $part2.Offset, ($part2.Size / 1GB))
|
||||
|
||||
Write-Host "`n=== 2. Reading Partition 2 Ext4 Superblock (Read-Only) ===" -ForegroundColor Cyan
|
||||
$pyCode = @"
|
||||
import struct, datetime
|
||||
p = r'$physDrive'
|
||||
offset = $($part2.Offset)
|
||||
with open(p, 'rb') as f:
|
||||
mbr = f.read(512)
|
||||
sig = struct.unpack('<I', mbr[0x1b8:0x1bc])[0]
|
||||
f.seek(offset + 1024)
|
||||
sb = f.read(1024)
|
||||
magic, state, errors = struct.unpack('<HHH', sb[0x38:0x3e])
|
||||
mnt_cnt, max_mnt = struct.unpack('<Hh', sb[0x34:0x38])
|
||||
last_mnt, last_wrt = struct.unpack('<II', sb[0x2c:0x34])
|
||||
last_chk = struct.unpack('<I', sb[0x40:0x44])[0]
|
||||
incompat = struct.unpack('<I', sb[0x60:0x64])[0]
|
||||
err_cnt = struct.unpack('<I', sb[0x194:0x198])[0]
|
||||
first_err_time = struct.unpack('<I', sb[0x198:0x19c])[0]
|
||||
last_err_time = struct.unpack('<I', sb[0x1c8:0x1cc])[0]
|
||||
fmt = lambda ts: datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') if ts else 'Never'
|
||||
print(f' MBR Disk Signature : {sig:08x} (PARTUUID={sig:08x}-02)')
|
||||
print(f' Ext4 Magic : 0x{magic:04x} (Valid={magic == 0xef53})')
|
||||
print(f' Filesystem State : {state} (1=Clean, 2=ERRORS_DETECTED, 4=Orphan_Recovery)')
|
||||
print(f' Journal Dirty Bit : {bool(incompat & 0x4)} (INCOMPAT_RECOVER)')
|
||||
print(f' Recorded FS Errors : {err_cnt} (First: {fmt(first_err_time)}, Last: {fmt(last_err_time)})')
|
||||
print(f' Errors Behavior : {errors} (1=Continue, 2=Remount-RO, 3=Panic)')
|
||||
print(f' Mount Count : {mnt_cnt} / {max_mnt}')
|
||||
print(f' Last Mount Time : {fmt(last_mnt)}')
|
||||
print(f' Last Write Time : {fmt(last_wrt)}')
|
||||
print(f' Last Fsck Time : {fmt(last_chk)}')
|
||||
"@
|
||||
python -c $pyCode
|
||||
}
|
||||
|
||||
Write-Host "`n=== 3. Verifying & Patching H:\cmdline.txt with 'fsck.repair=yes ro' ===" -ForegroundColor Cyan
|
||||
$cmdlinePath = "H:\cmdline.txt"
|
||||
if (Test-Path $cmdlinePath) {
|
||||
$orig = (Get-Content $cmdlinePath -Raw).Trim()
|
||||
Write-Host " Current H:\cmdline.txt: $orig"
|
||||
$tokens = [System.Collections.Generic.List[string]]($orig -split '\s+')
|
||||
$changed = $false
|
||||
if ($tokens.Contains("rw")) {
|
||||
$null = $tokens.Remove("rw")
|
||||
$changed = $true
|
||||
}
|
||||
if (-not $tokens.Contains("fsck.repair=yes")) {
|
||||
$idx = $tokens.FindIndex({ param($t) $t.StartsWith("rootfstype=") })
|
||||
if ($idx -ge 0) { $tokens.Insert($idx + 1, "fsck.repair=yes") } else { $tokens.Add("fsck.repair=yes") }
|
||||
$changed = $true
|
||||
}
|
||||
if (-not $tokens.Contains("fsck.mode=force")) {
|
||||
$idx = $tokens.IndexOf("fsck.repair=yes")
|
||||
if ($idx -ge 0) { $tokens.Insert($idx + 1, "fsck.mode=force") } else { $tokens.Add("fsck.mode=force") }
|
||||
$changed = $true
|
||||
}
|
||||
if (-not $tokens.Contains("ro")) {
|
||||
$idx = $tokens.IndexOf("rootwait")
|
||||
if ($idx -ge 0) { $tokens.Insert($idx, "ro") } else { $tokens.Add("ro") }
|
||||
$changed = $true
|
||||
}
|
||||
if ($changed) {
|
||||
Copy-Item $cmdlinePath "H:\cmdline.txt.bak" -Force
|
||||
$newCmdline = ($tokens -join " ") + "`n"
|
||||
[System.IO.File]::WriteAllText($cmdlinePath, $newCmdline, (New-Object System.Text.UTF8Encoding($false)))
|
||||
Write-Host " Patched H:\cmdline.txt: $($newCmdline.Trim())" -ForegroundColor Green
|
||||
} else {
|
||||
Write-Host " Verified H:\cmdline.txt already has 'fsck.repair=yes' and 'ro'." -ForegroundColor Green
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "`n=== Ready! Safe to eject H: and boot the Raspberry Pi. ===" -ForegroundColor Green
|
||||
Write-Host "With 'fsck.repair=yes ro' in H:\cmdline.txt, the Pi kernel mounts / read-only first,"
|
||||
Write-Host "runs 'fsck.ext4 -y' on Partition 2 automatically during boot, and then remounts / read-write."
|
||||
96
scripts/setup_gotify_channels.py
Normal file
96
scripts/setup_gotify_channels.py
Normal file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Provision Gotify Users, Broadcast Applications, and Device Client Tokens
|
||||
on Snarfnet (http://140.44.4.70:8090).
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import urllib.request
|
||||
import paramiko
|
||||
|
||||
SWARM_MANAGER = "140.44.4.71"
|
||||
SSH_USER = "snarf"
|
||||
SSH_PASS = "Katzir476!"
|
||||
GOTIFY_URL = "http://140.44.4.70:8090"
|
||||
|
||||
APPLICATIONS = [
|
||||
(1, "A4uN4fEmErGeNcYtOkEn", 1, "UPS & Power Resilience", "NUT Master UPS power events and tiered shutdown broadcasts", 0, "", 9),
|
||||
(2, "A7zBx9InFrAaLeRtStOk", 1, "Zabbix Infrastructure Alerts", "Zabbix 7.0 infrastructure triggers, node health, and threshold alarms", 0, "", 7),
|
||||
(3, "A3hOmEaSsIsTaNtToKeN", 1, "Home Assistant Smart Home", "Home Assistant smart home alerts and automation broadcasts", 0, "", 5),
|
||||
(4, "A8aWxGiToPsAuToMaTiO", 1, "AWX & GitOps Automation", "Ansible AWX, Semaphore, and ArgoCD deployment & patching notifications", 0, "", 4),
|
||||
(5, "A6bAcKuPsAnDsToRaGeH", 1, "Backups & Storage Health", "NPM, Gotify, Homebox, and Longhorn NFS backup verification alerts", 0, "", 6),
|
||||
]
|
||||
|
||||
CLIENTS = [
|
||||
(1, "C1dIsCoRdBrIdGeClIeN", 1, "Gotify-Discord Multi-Channel Bridge"),
|
||||
(2, "C2sNaRfMoBiLePhOnEcL", 1, "Snarf Primary Mobile Phone"),
|
||||
(3, "C3sNaRfTaBlEtDeViCeC", 1, "Snarf Tablet / Secondary Device"),
|
||||
(4, "C4hOmEpAgEdAsHbOaRdC", 1, "Homepage Dashboard & Workstation"),
|
||||
]
|
||||
|
||||
DB_PATHS = [
|
||||
"/var/lib/docker/volumes/gotify_gotify_data/_data/gotify.db",
|
||||
"/mnt/swarm_shared/legacy_volumes/gotify/gotify.db",
|
||||
"/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db",
|
||||
]
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(SWARM_MANAGER, username=SSH_USER, password=SSH_PASS, timeout=10)
|
||||
|
||||
remote_py = f"""echo '{SSH_PASS}' | sudo -S python3 -c '
|
||||
import sqlite3, bcrypt
|
||||
|
||||
pw_hash = bcrypt.hashpw(b"Katzir476!", bcrypt.gensalt(rounds=10)).replace(b"$2b$", b"$2a$")
|
||||
|
||||
for db_path in {json.dumps(DB_PATHS)}:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("UPDATE users SET pass = ?, admin = 1 WHERE id = 1", (pw_hash,))
|
||||
cur.execute("INSERT OR REPLACE INTO users (id, name, pass, admin) VALUES (2, ?, ?, 1)", ("snarf", pw_hash))
|
||||
|
||||
for app in {json.dumps(APPLICATIONS)}:
|
||||
cur.execute(\"\"\"
|
||||
INSERT OR REPLACE INTO applications
|
||||
(id, token, user_id, name, description, internal, image, default_priority)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
\"\"\", app)
|
||||
|
||||
for cl in {json.dumps(CLIENTS)}:
|
||||
cur.execute(\"\"\"
|
||||
INSERT OR REPLACE INTO clients
|
||||
(id, token, user_id, name)
|
||||
VALUES (?, ?, ?, ?)
|
||||
\"\"\", cl)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print("Configured:", db_path)
|
||||
'"""
|
||||
stdin, stdout, stderr = ssh.exec_command(remote_py)
|
||||
print(stdout.read().decode("utf-8", errors="ignore"))
|
||||
err = stderr.read().decode("utf-8", errors="ignore")
|
||||
if err:
|
||||
print("STDERR:", err)
|
||||
ssh.close()
|
||||
|
||||
# Verify via Gotify HTTP API
|
||||
b64 = base64.b64encode(b"admin:Katzir476!").decode("ascii")
|
||||
req = urllib.request.Request(f"{GOTIFY_URL}/application", headers={"Authorization": f"Basic {b64}"})
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
apps_resp = json.loads(r.read().decode("utf-8"))
|
||||
print(f"Verified {len(apps_resp)} Gotify Applications via REST API:")
|
||||
for a in apps_resp:
|
||||
print(f" - [{a['id']}] {a['name']} (token={a['token']}, defaultPriority={a['defaultPriority']})")
|
||||
|
||||
req_c = urllib.request.Request(f"{GOTIFY_URL}/client", headers={"Authorization": f"Basic {b64}"})
|
||||
with urllib.request.urlopen(req_c, timeout=8) as r:
|
||||
clients_resp = json.loads(r.read().decode("utf-8"))
|
||||
print(f"Verified {len(clients_resp)} Gotify Clients via REST API:")
|
||||
for c in clients_resp:
|
||||
print(f" - [{c['id']}] {c['name']} (token={c['token']})")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
245
scripts/setup_homeassistant_notifications.py
Normal file
245
scripts/setup_homeassistant_notifications.py
Normal file
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Configure Home Assistant (14.4.4.20:8123) Notification Forwarding & Infrastructure Automations:
|
||||
1. Updates gotify-discord-bridge on 140.44.4.71 with Home Assistant logbook.log watcher.
|
||||
2. Creates 'script.snarfnet_broadcast' in Home Assistant for unified Gotify + Discord + Mobile App alerts.
|
||||
3. Creates 'Snarfnet UPS Power & Exhaust Protocol' and 'Snarfnet Server Room Thermal Guard' automations.
|
||||
4. Enhances the 4 Bambu H2D 3D Printer automations to also broadcast to Gotify & Discord #home-assistant.
|
||||
5. Triggers a live end-to-end verification notification with real-time Server Room & UPS telemetry.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
HA_URL = "http://14.4.4.20:8123"
|
||||
HA_TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkZWNlMWI5ZjJhZDQ0MTRmODU4NGYzN2I1YjgwZDQxMiIsImlhdCI6MTc5MzQ1MjQ2MywiZXhwIjoyMTA4ODEyNDYzfQ.JznpOqsCSnHz6aTVa8K9pQCz1816DC8SrmS1xn1Sm2k"
|
||||
|
||||
|
||||
def ha_req(path, method="GET", data=None):
|
||||
req = urllib.request.Request(
|
||||
f"{HA_URL}{path}",
|
||||
data=json.dumps(data).encode("utf-8") if data is not None else None,
|
||||
headers={
|
||||
"Authorization": f"Bearer {HA_TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method=method,
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=12) as r:
|
||||
body = r.read().decode("utf-8")
|
||||
return json.loads(body) if body else {}
|
||||
|
||||
|
||||
def main():
|
||||
# 1. Update discord_bridge.py on 140.44.4.71 and restart bridge container
|
||||
with open(r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\scripts\gotify_discord_bridge.py", "r", encoding="utf-8") as f:
|
||||
bridge_code = f.read()
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.file("/tmp/discord_bridge.py", "w") as rf:
|
||||
rf.write(bridge_code)
|
||||
sftp.close()
|
||||
|
||||
cmd = r"""echo 'Katzir476!' | sudo -S bash -c '
|
||||
rm -f /var/lib/docker/volumes/gotify_gotify_data/_data/ha_last_logbook_when.txt /mnt/swarm_shared/legacy_volumes/gotify/ha_last_logbook_when.txt
|
||||
cp /tmp/discord_bridge.py /var/lib/docker/volumes/gotify_gotify_data/_data/discord_bridge.py
|
||||
cp /tmp/discord_bridge.py /mnt/swarm_shared/legacy_volumes/gotify/discord_bridge.py
|
||||
chmod 755 /var/lib/docker/volumes/gotify_gotify_data/_data/discord_bridge.py /mnt/swarm_shared/legacy_volumes/gotify/discord_bridge.py
|
||||
docker service scale gotify_gotify-discord-bridge=0
|
||||
docker rm -f $(docker ps -aq -f name=gotify_gotify-discord-bridge) 2>/dev/null || true
|
||||
docker service scale gotify_gotify-discord-bridge=1
|
||||
'"""
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
stdout.read()
|
||||
print("Updated and restarted gotify_gotify-discord-bridge on 140.44.4.71.")
|
||||
|
||||
# 2. Provision script.snarfnet_broadcast in Home Assistant
|
||||
broadcast_script = {
|
||||
"alias": "Snarfnet Broadcast Notification",
|
||||
"description": "Broadcasts alerts to Gotify, Discord #home-assistant webhook, and Pixel Companion Apps",
|
||||
"mode": "parallel",
|
||||
"fields": {
|
||||
"title": {"description": "Notification title", "example": "Smart Home Alert"},
|
||||
"message": {"description": "Notification body", "example": "Event details"},
|
||||
"notification_id": {"description": "Unique notification key", "example": "snarfnet_alert"},
|
||||
},
|
||||
"sequence": [
|
||||
{
|
||||
"action": "logbook.log",
|
||||
"data": {
|
||||
"name": "{{ title | default('Home Assistant Alert') }}",
|
||||
"message": "{{ message }}",
|
||||
"entity_id": "script.snarfnet_broadcast",
|
||||
},
|
||||
},
|
||||
{
|
||||
"action": "persistent_notification.create",
|
||||
"data": {
|
||||
"title": "{{ title | default('Home Assistant Alert') }}",
|
||||
"message": "{{ message }}",
|
||||
"notification_id": "{{ notification_id | default('snarfnet_' ~ (now().timestamp() | int)) }}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"action": "notify.mobile_app_pixel_10_pro_fold",
|
||||
"data": {
|
||||
"title": "{{ title | default('Home Assistant Alert') }}",
|
||||
"message": "{{ message }}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"action": "notify.mobile_app_pixel_9_pro_fold",
|
||||
"data": {
|
||||
"title": "{{ title | default('Home Assistant Alert') }}",
|
||||
"message": "{{ message }}",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
ha_req("/api/config/script/config/snarfnet_broadcast", method="POST", data=broadcast_script)
|
||||
print("Configured script.snarfnet_broadcast in Home Assistant.")
|
||||
|
||||
# 3. Provision UPS Power State & Exhaust Fan Automation
|
||||
ups_automation = {
|
||||
"id": "1793453000001",
|
||||
"alias": "Snarfnet UPS Power & Exhaust Protocol",
|
||||
"description": "Broadcasts UPS state transitions to Gotify/Discord/Mobile and engages Server Closet exhaust fan on battery",
|
||||
"triggers": [
|
||||
{
|
||||
"trigger": "state",
|
||||
"entity_id": ["sensor.server_closet_status_data"],
|
||||
}
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"condition": "template",
|
||||
"value_template": "{{ trigger.from_state.state != trigger.to_state.state and trigger.to_state.state not in ['unknown', 'unavailable'] }}",
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"action": "script.snarfnet_broadcast",
|
||||
"data": {
|
||||
"title": "⚡ UPS Status Changed: {{ states('sensor.server_closet_status') }} ({{ states('sensor.server_closet_status_data') }})",
|
||||
"message": "Server Closet UPS transitioned from {{ trigger.from_state.state }} to {{ trigger.to_state.state }}. Battery: {{ states('sensor.server_closet_battery_charge') }}% | Load: {{ states('sensor.server_closet_load') }}% | Input: {{ states('sensor.server_closet_input_voltage') }}V | Closet Temp: {{ states('sensor.server_room_temperature_temperature') }}°F.",
|
||||
"notification_id": "snarfnet_ups_state",
|
||||
},
|
||||
},
|
||||
{
|
||||
"choose": [
|
||||
{
|
||||
"conditions": [
|
||||
{
|
||||
"condition": "template",
|
||||
"value_template": "{{ 'OB' in states('sensor.server_closet_status_data') or 'LB' in states('sensor.server_closet_status_data') }}",
|
||||
}
|
||||
],
|
||||
"sequence": [
|
||||
{
|
||||
"action": "fan.set_percentage",
|
||||
"target": {"entity_id": "fan.server_closet_switch"},
|
||||
"data": {"percentage": 100},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
],
|
||||
"mode": "queued",
|
||||
}
|
||||
ha_req("/api/config/automation/config/1793453000001", method="POST", data=ups_automation)
|
||||
print("Configured automation 'Snarfnet UPS Power & Exhaust Protocol' (1793453000001).")
|
||||
|
||||
# 4. Provision Server Room Thermal Guard Automation
|
||||
thermal_automation = {
|
||||
"id": "1793453000002",
|
||||
"alias": "Snarfnet Server Room Thermal Guard",
|
||||
"description": "Automatically engages Server Closet exhaust fan and broadcasts alert if Server Room exceeds 82°F",
|
||||
"triggers": [
|
||||
{
|
||||
"trigger": "numeric_state",
|
||||
"entity_id": ["sensor.server_room_temperature_temperature"],
|
||||
"above": 82,
|
||||
"for": {"hours": 0, "minutes": 2, "seconds": 0},
|
||||
}
|
||||
],
|
||||
"conditions": [],
|
||||
"actions": [
|
||||
{
|
||||
"action": "fan.set_percentage",
|
||||
"target": {"entity_id": "fan.server_closet_switch"},
|
||||
"data": {"percentage": 100},
|
||||
},
|
||||
{
|
||||
"action": "script.snarfnet_broadcast",
|
||||
"data": {
|
||||
"title": "🌡️ High Server Room Temperature Warning",
|
||||
"message": "Server Room temperature has reached {{ states('sensor.server_room_temperature_temperature') }}°F (Humidity: {{ states('sensor.server_room_temperature_humidity') }}%). Server Closet exhaust fan engaged at 100%.",
|
||||
"notification_id": "snarfnet_thermal_warning",
|
||||
},
|
||||
},
|
||||
],
|
||||
"mode": "single",
|
||||
}
|
||||
ha_req("/api/config/automation/config/1793453000002", method="POST", data=thermal_automation)
|
||||
print("Configured automation 'Snarfnet Server Room Thermal Guard' (1793453000002).")
|
||||
|
||||
# 5. Enhance the 4 existing 3D Printer automations to call script.snarfnet_broadcast
|
||||
printer_autos = [
|
||||
("1788367985433", "🖨️ 3D Print Started", "A 3D print has started on Bambu H2D.", "h2d_print_start"),
|
||||
("1788368709567", "✅ 3D Print Completed", "A 3D print has finished on Bambu H2D.", "h2d_print_complete"),
|
||||
("1788370272217", "⚠️ 3D Print Stalled", "A 3D print has stalled/paused on Bambu H2D!", "h2d_print_stalled"),
|
||||
("1788370321997", "🚨 3D Print Failed", "A 3D print has failed on Bambu H2D!", "h2d_print_failed"),
|
||||
]
|
||||
for aid, title, msg, nid in printer_autos:
|
||||
cfg = ha_req(f"/api/config/automation/config/{aid}")
|
||||
# Filter out old persistent_notification/snarfnet_broadcast actions and append clean script.snarfnet_broadcast
|
||||
actions = [a for a in cfg.get("actions", []) if a.get("action") not in ("persistent_notification.create", "script.snarfnet_broadcast")]
|
||||
actions.append(
|
||||
{
|
||||
"action": "script.snarfnet_broadcast",
|
||||
"metadata": {},
|
||||
"data": {
|
||||
"title": title,
|
||||
"message": msg,
|
||||
"notification_id": nid,
|
||||
},
|
||||
}
|
||||
)
|
||||
cfg["actions"] = actions
|
||||
ha_req(f"/api/config/automation/config/{aid}", method="POST", data=cfg)
|
||||
print(f"Enhanced 3D Printer automation [{aid}] '{cfg.get('alias')}' with script.snarfnet_broadcast.")
|
||||
|
||||
ha_req("/api/services/script/reload", method="POST", data={})
|
||||
ha_req("/api/services/automation/reload", method="POST", data={})
|
||||
|
||||
# Wait 5s for restarted bridge container to finish [INIT-HA]
|
||||
time.sleep(5)
|
||||
|
||||
# 6. Trigger live test via script.snarfnet_broadcast
|
||||
ha_req(
|
||||
"/api/services/script/snarfnet_broadcast",
|
||||
method="POST",
|
||||
data={
|
||||
"title": "Home Assistant <-> Gotify <-> Discord Linked",
|
||||
"message": "Live Telemetry: Server Room {{ states('sensor.server_room_temperature_temperature') }}°F ({{ states('sensor.server_room_temperature_humidity') }}% RH) | UPS: {{ states('sensor.server_closet_status') }} ({{ states('sensor.server_closet_battery_charge') }}% Battery, {{ states('sensor.server_closet_load') }}% Load, {{ states('sensor.server_closet_input_voltage') }}V).",
|
||||
"notification_id": "snarfnet_ha_bridge_verified",
|
||||
},
|
||||
)
|
||||
print("Triggered live script.snarfnet_broadcast test in Home Assistant.")
|
||||
|
||||
time.sleep(5)
|
||||
stdin, stdout, stderr = ssh.exec_command("echo 'Katzir476!' | sudo -S docker logs $(docker ps -q -f name=gotify_gotify-discord-bridge) --tail 25 2>&1")
|
||||
print("Bridge logs:\n", stdout.read().decode("utf-8", errors="ignore"))
|
||||
ssh.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
205
scripts/setup_zabbix_notifications.py
Normal file
205
scripts/setup_zabbix_notifications.py
Normal file
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Configure Zabbix 7.0.16 Notification Pipeline:
|
||||
1. Creates/Updates 'Snarfnet Gotify + Discord Failover' Webhook MediaType.
|
||||
- Primary path: Sends to Gotify (http://140.44.4.70:8090) -> pushes to devices + Gotify-Discord Bridge.
|
||||
- Failover path: If Gotify/Shipyard is unreachable, sends directly to Discord (#infra-alerts or #ups-power).
|
||||
2. Attaches MediaType to Zabbix 'Admin' user (userid=1).
|
||||
3. Enables Trigger Action 'Report problems to Zabbix administrators' (status=0).
|
||||
"""
|
||||
import json
|
||||
import ssl
|
||||
import urllib.request
|
||||
|
||||
ZABBIX_URL = "https://zabbix.snarfnet.net/api_jsonrpc.php"
|
||||
ZABBIX_USER = "Admin"
|
||||
ZABBIX_PASS = "zabbix"
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
|
||||
def zbx_req(method: str, params: dict, auth: str = None):
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"method": method,
|
||||
"params": params,
|
||||
"id": 1,
|
||||
}
|
||||
headers = {"Content-Type": "application/json-rpc"}
|
||||
if auth:
|
||||
headers["Authorization"] = f"Bearer {auth}"
|
||||
req = urllib.request.Request(
|
||||
ZABBIX_URL,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
|
||||
resp = json.loads(r.read().decode("utf-8"))
|
||||
if "error" in resp:
|
||||
raise RuntimeError(f"Zabbix API error in {method}: {resp['error']}")
|
||||
return resp["result"]
|
||||
|
||||
|
||||
WEBHOOK_JS = r"""
|
||||
try {
|
||||
var params = JSON.parse(value);
|
||||
var subject = params.Subject || 'Zabbix Alert';
|
||||
var message = params.Message || '';
|
||||
var severity = parseInt(params.Severity || '3', 10);
|
||||
var host = (params.Host || '').toLowerCase();
|
||||
|
||||
// Map Zabbix severity (0-5) to Gotify priority (2-10)
|
||||
var prioMap = {0: 2, 1: 3, 2: 5, 3: 7, 4: 9, 5: 10};
|
||||
var priority = prioMap[severity] || 6;
|
||||
|
||||
var isUps = (host.indexOf('nut') !== -1 || host.indexOf('ups') !== -1 || subject.toLowerCase().indexOf('ups') !== -1);
|
||||
var gotifyToken = isUps ? params.GotifyUpsToken : params.GotifyInfraToken;
|
||||
var discordWebhook = isUps ? params.DiscordUpsWebhook : params.DiscordInfraWebhook;
|
||||
|
||||
var req = new HttpRequest();
|
||||
req.addHeader('Content-Type: application/json');
|
||||
|
||||
var gotifyUrl = params.GotifyUrl.replace(/\/$/, '') + '/message?token=' + gotifyToken;
|
||||
var gotifyPayload = JSON.stringify({
|
||||
title: subject,
|
||||
message: message,
|
||||
priority: priority
|
||||
});
|
||||
|
||||
var resp = req.post(gotifyUrl, gotifyPayload);
|
||||
if (req.getStatus() >= 200 && req.getStatus() < 300) {
|
||||
return 'OK (Delivered via Gotify + Discord Bridge)';
|
||||
}
|
||||
|
||||
// Automatic Direct-to-Discord Failover if Gotify/Shipyard is unreachable
|
||||
Zabbix.log(3, '[Snarfnet Webhook] Gotify returned HTTP ' + req.getStatus() + ', failing over directly to Discord.');
|
||||
var dReq = new HttpRequest();
|
||||
dReq.addHeader('Content-Type: application/json');
|
||||
var color = (priority >= 8) ? 15158332 : ((priority >= 5) ? 15965202 : 3447003);
|
||||
var discordPayload = JSON.stringify({
|
||||
username: 'Snarfnet Zabbix (Direct Failover)',
|
||||
embeds: [{
|
||||
title: subject.substring(0, 256),
|
||||
description: message.substring(0, 4000),
|
||||
color: color,
|
||||
footer: {text: 'Zabbix Direct Failover • Priority ' + priority}
|
||||
}]
|
||||
});
|
||||
dReq.post(discordWebhook, discordPayload);
|
||||
if (dReq.getStatus() >= 200 && dReq.getStatus() < 300) {
|
||||
return 'OK (Delivered via Direct Discord Failover)';
|
||||
}
|
||||
throw 'Both Gotify and Direct Discord failed (Discord HTTP ' + dReq.getStatus() + ')';
|
||||
} catch (error) {
|
||||
Zabbix.log(3, '[Snarfnet Webhook] Error: ' + error);
|
||||
throw 'Sending failed: ' + error;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
token = zbx_req("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS})
|
||||
print("Authenticated to Zabbix 7.0.16 API.")
|
||||
|
||||
mt_name = "Snarfnet Gotify + Discord Failover"
|
||||
existing = zbx_req("mediatype.get", {"filter": {"name": [mt_name]}}, auth=token)
|
||||
|
||||
mt_params = [
|
||||
{"name": "Subject", "value": "{ALERT.SUBJECT}"},
|
||||
{"name": "Message", "value": "{ALERT.MESSAGE}"},
|
||||
{"name": "Severity", "value": "{EVENT.NSEVERITY}"},
|
||||
{"name": "Host", "value": "{HOST.NAME}"},
|
||||
{"name": "GotifyUrl", "value": "http://140.44.4.70:8090"},
|
||||
{"name": "GotifyInfraToken", "value": "A7zBx9InFrAaLeRtStOk"},
|
||||
{"name": "GotifyUpsToken", "value": "A4uN4fEmErGeNcYtOkEn"},
|
||||
{
|
||||
"name": "DiscordInfraWebhook",
|
||||
"value": "https://discord.com/api/webhooks/1566076777963593772/wKN30N0ZXVJKGP-prZ1_P9cp_u7Oakmh8R4bVh0qr8RFpyeGcJZrHfPQBJxtfzQqb4dn",
|
||||
},
|
||||
{
|
||||
"name": "DiscordUpsWebhook",
|
||||
"value": "https://discord.com/api/webhooks/1566076118535114813/2e_fCJeHb-mF8VwerzOORGRa-QUDFuwN6R2Gfz-MGUiIOudA0PiXGZSyd19PNFG-vR7D",
|
||||
},
|
||||
]
|
||||
|
||||
mt_templates = [
|
||||
{
|
||||
"eventsource": 0, # Triggers
|
||||
"recovery": 0, # Problem
|
||||
"subject": "🚨 [{EVENT.SEVERITY}] {HOST.NAME}: {EVENT.NAME}",
|
||||
"message": "**Host:** `{HOST.NAME}` (`{HOST.IP}`)\n**Severity:** {EVENT.SEVERITY}\n**Trigger:** {EVENT.NAME}\n**Status:** {EVENT.STATUS}\n**Time:** {EVENT.DATE} {EVENT.TIME}\n**Event ID:** {EVENT.ID}",
|
||||
},
|
||||
{
|
||||
"eventsource": 0, # Triggers
|
||||
"recovery": 1, # Recovery
|
||||
"subject": "✅ [RESOLVED] {HOST.NAME}: {EVENT.NAME}",
|
||||
"message": "**Host:** `{HOST.NAME}` (`{HOST.IP}`)\n**Resolved After:** {EVENT.DURATION}\n**Trigger:** {EVENT.NAME}\n**Time:** {EVENT.RECOVERY.DATE} {EVENT.RECOVERY.TIME}\n**Event ID:** {EVENT.ID}",
|
||||
},
|
||||
]
|
||||
|
||||
if existing:
|
||||
mt_id = existing[0]["mediatypeid"]
|
||||
zbx_req(
|
||||
"mediatype.update",
|
||||
{
|
||||
"mediatypeid": mt_id,
|
||||
"type": 4, # Webhook
|
||||
"status": 0, # Enabled
|
||||
"script": WEBHOOK_JS.strip(),
|
||||
"timeout": "15s",
|
||||
"parameters": mt_params,
|
||||
"message_templates": mt_templates,
|
||||
},
|
||||
auth=token,
|
||||
)
|
||||
print(f"Updated MediaType '{mt_name}' (mediatypeid={mt_id})")
|
||||
else:
|
||||
res = zbx_req(
|
||||
"mediatype.create",
|
||||
{
|
||||
"name": mt_name,
|
||||
"type": 4, # Webhook
|
||||
"status": 0, # Enabled
|
||||
"script": WEBHOOK_JS.strip(),
|
||||
"timeout": "15s",
|
||||
"parameters": mt_params,
|
||||
"message_templates": mt_templates,
|
||||
},
|
||||
auth=token,
|
||||
)
|
||||
mt_id = res["mediatypeids"][0]
|
||||
print(f"Created MediaType '{mt_name}' (mediatypeid={mt_id})")
|
||||
|
||||
# Attach to Admin user (userid=1) for Warning, Average, High, Disaster (bitmask 60 = 4+8+16+32)
|
||||
zbx_req(
|
||||
"user.update",
|
||||
{
|
||||
"userid": "1",
|
||||
"medias": [
|
||||
{
|
||||
"mediatypeid": str(mt_id),
|
||||
"sendto": "snarf@snarfnet.net",
|
||||
"active": 0,
|
||||
"severity": 60, # Warning (2), Average (3), High (4), Disaster (5)
|
||||
"period": "1-7,00:00-24:00",
|
||||
}
|
||||
],
|
||||
},
|
||||
auth=token,
|
||||
)
|
||||
print("Attached MediaType to Zabbix Admin user (userid=1).")
|
||||
|
||||
# Enable Trigger Action 'Report problems to Zabbix administrators'
|
||||
actions = zbx_req("action.get", {"filter": {"eventsource": 0}}, auth=token)
|
||||
for act in actions:
|
||||
if "Report problems" in act["name"] or act["actionid"] == "3":
|
||||
zbx_req("action.update", {"actionid": act["actionid"], "status": 0}, auth=token)
|
||||
print(f"Enabled Trigger Action '{act['name']}' (actionid={act['actionid']}).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
112
scripts/wsl_repair_ext4.sh
Normal file
112
scripts/wsl_repair_ext4.sh
Normal file
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
echo "=== 1. Detecting Attached RPi SD/SSD in WSL2 ==="
|
||||
lsblk -o NAME,SIZE,FSTYPE,PARTUUID,MODEL
|
||||
|
||||
# Find the newly attached disk (not sda, sdb, sdc, sdd)
|
||||
TARGET_DEV=""
|
||||
for dev in /dev/sd[e-z]; do
|
||||
if [ -b "${dev}2" ]; then
|
||||
TARGET_DEV="$dev"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$TARGET_DEV" ]; then
|
||||
echo "[ERROR] Could not find attached RPi disk (/dev/sde..z with partition 2)!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BOOT_PART="${TARGET_DEV}1"
|
||||
ROOT_PART="${TARGET_DEV}2"
|
||||
echo "Target Disk: $TARGET_DEV (Boot: $BOOT_PART, Root: $ROOT_PART)"
|
||||
blkid "$BOOT_PART" "$ROOT_PART"
|
||||
|
||||
echo ""
|
||||
echo "=== 2. Inspecting Ext4 Superblock BEFORE Repair ($ROOT_PART) ==="
|
||||
dumpe2fs -h "$ROOT_PART" 2>/dev/null | grep -E "Filesystem volume name|Filesystem UUID|Filesystem state|Errors behavior|Filesystem created|Last mount time|Last write time|Mount count|Maximum mount count|Last checked|Journal|Inode count|Free inodes|Block count|Free blocks"
|
||||
|
||||
echo ""
|
||||
echo "=== 3. Running e2fsck -fy on $ROOT_PART ==="
|
||||
e2fsck -fy "$ROOT_PART"
|
||||
FSCK_RC=$?
|
||||
echo "e2fsck exit code: $FSCK_RC (0=no errors, 1=errors corrected, 2=corrected)"
|
||||
|
||||
echo ""
|
||||
echo "=== 4. Setting Ext4 Errors Behavior to 'continue' ==="
|
||||
tune2fs -e continue "$ROOT_PART"
|
||||
dumpe2fs -h "$ROOT_PART" 2>/dev/null | grep -E "Filesystem state|Errors behavior|Last checked"
|
||||
|
||||
echo ""
|
||||
echo "=== 5. Mounting $ROOT_PART & $BOOT_PART to Inspect & Harden OS Config ==="
|
||||
mkdir -p /mnt/rpi_root /mnt/rpi_boot
|
||||
mount "$ROOT_PART" /mnt/rpi_root
|
||||
mount "$BOOT_PART" /mnt/rpi_boot
|
||||
|
||||
echo "--- Hostname & Network Config ---"
|
||||
cat /mnt/rpi_root/etc/hostname 2>/dev/null || true
|
||||
cat /mnt/rpi_root/etc/network/interfaces 2>/dev/null | grep -v "^#" | grep -v "^$" || true
|
||||
ip_cfg=$(grep -rn "140.44.4." /mnt/rpi_root/etc/network/ /mnt/rpi_root/etc/NetworkManager/ /mnt/rpi_root/etc/systemd/network/ 2>/dev/null || true)
|
||||
echo "$ip_cfg"
|
||||
|
||||
echo "--- Current /etc/fstab ---"
|
||||
cat /mnt/rpi_root/etc/fstab
|
||||
|
||||
# Harden /etc/fstab (add nofail to /boot/firmware or /boot and any nfs/glusterfs mounts)
|
||||
cp /mnt/rpi_root/etc/fstab /mnt/rpi_root/etc/fstab.bak
|
||||
python3 -c '
|
||||
p = "/mnt/rpi_root/etc/fstab"
|
||||
with open(p, "r") as f:
|
||||
lines = f.readlines()
|
||||
new_lines = []
|
||||
for line in lines:
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#"):
|
||||
new_lines.append(line)
|
||||
continue
|
||||
parts = s.split()
|
||||
if len(parts) >= 4:
|
||||
mp, fstype, opts = parts[1], parts[2], parts[3]
|
||||
if (mp in ("/boot/firmware", "/boot") or fstype == "vfat") and "nofail" not in opts:
|
||||
parts[3] = opts + ",nofail"
|
||||
line = "\t".join(parts) + "\n"
|
||||
elif fstype in ("glusterfs", "nfs", "nfs4", "cifs") and "nofail" not in opts:
|
||||
parts[3] = opts + ",nofail,x-systemd.automount,x-systemd.device-timeout=30"
|
||||
line = "\t".join(parts) + "\n"
|
||||
new_lines.append(line)
|
||||
with open(p, "w") as f:
|
||||
f.writelines(new_lines)
|
||||
'
|
||||
echo "--- Hardened /etc/fstab ---"
|
||||
cat /mnt/rpi_root/etc/fstab
|
||||
|
||||
# Ensure both /mnt/rpi_boot/cmdline.txt and /mnt/rpi_root/boot/cmdline.txt (if regular file) have fsck.repair=yes and ro
|
||||
for cpath in /mnt/rpi_boot/cmdline.txt /mnt/rpi_root/boot/cmdline.txt /mnt/rpi_root/boot/firmware/cmdline.txt; do
|
||||
if [ -f "$cpath" ] && [ ! -L "$cpath" ]; then
|
||||
if ! grep -q "fsck.repair=yes" "$cpath"; then
|
||||
sed -i "s/rootfstype=ext4/rootfstype=ext4 fsck.repair=yes/" "$cpath"
|
||||
fi
|
||||
if ! grep -qE "(^| )ro( |$)" "$cpath"; then
|
||||
sed -i "s/rootwait/ro rootwait/" "$cpath"
|
||||
fi
|
||||
echo "Verified $cpath: $(cat "$cpath")"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- Checking Kernel Modules in /lib/modules vs Boot Kernel ---"
|
||||
ls -la /mnt/rpi_root/lib/modules/
|
||||
ls -la /mnt/rpi_root/boot/
|
||||
|
||||
echo "--- Checking Recent Logs / Crash Evidence on Root FS ---"
|
||||
ls -la /mnt/rpi_root/var/log/ 2>/dev/null | head -n 20
|
||||
if [ -d /mnt/rpi_root/var/log/journal ]; then
|
||||
journalctl -D /mnt/rpi_root/var/log/journal -n 30 --no-pager 2>/dev/null || true
|
||||
fi
|
||||
tail -n 30 /mnt/rpi_root/var/log/syslog /mnt/rpi_root/var/log/messages /mnt/rpi_root/var/log/daemon.log 2>/dev/null || true
|
||||
|
||||
sync
|
||||
umount /mnt/rpi_boot
|
||||
umount /mnt/rpi_root
|
||||
echo "=== 6. Repair and Hardening Complete! Unmounted cleanly. ==="
|
||||
|
||||
Reference in New Issue
Block a user