From 336b7759d05c15825d61a34d810d49ebeec331c1 Mon Sep 17 00:00:00 2001 From: Snarf Date: Thu, 12 Nov 2026 20:18:37 -0500 Subject: [PATCH] feat: add infrastructure automation and diagnostic scripts for Zabbix, Nginx, and Authentik --- scripts/add_forward_rules.py | 22 ++++++++++ scripts/add_media_to_snarf.py | 32 +++++++++++++++ scripts/apply_h21_authentik.py | 42 +++++++++++-------- scripts/audit_zabbix_alerts.py | 48 ++++++++++++++++++++++ scripts/cat_21_conf.py | 1 + scripts/check_alerts_status.py | 19 +++++++++ scripts/check_authentik_proxy_providers.py | 1 + scripts/check_homepage_html.py | 1 + scripts/check_nat.py | 20 +++++++++ scripts/check_npm_ports.py | 20 +++++++++ scripts/check_npm_streams.py | 11 +++++ scripts/check_one_problem.py | 24 +++++++++++ scripts/check_remaining_agent_problems.py | 23 +++++++++++ scripts/check_two_events.py | 19 +++++++++ scripts/check_zabbix_cluster.py | 23 +++++++++++ scripts/copy_topology_to_npm.py | 43 +++++++++---------- scripts/diagnose_zabbix.py | 22 ++++++++++ scripts/fix_swm01_output.py | 24 +++++++++++ scripts/inspect_assets_conf.py | 13 ++++++ scripts/inspect_npm_adv.py | 1 + scripts/inspect_npm_container.py | 1 + scripts/persist_zabbix_nat.py | 39 ++++++++++++++++++ scripts/test_agent_dns.py | 21 ++++++++++ scripts/test_from_ctl01.py | 21 ++++++++++ scripts/test_zabbix_nat.py | 22 ++++++++++ 25 files changed, 472 insertions(+), 41 deletions(-) create mode 100644 scripts/add_forward_rules.py create mode 100644 scripts/add_media_to_snarf.py create mode 100644 scripts/audit_zabbix_alerts.py create mode 100644 scripts/check_alerts_status.py create mode 100644 scripts/check_nat.py create mode 100644 scripts/check_npm_ports.py create mode 100644 scripts/check_npm_streams.py create mode 100644 scripts/check_one_problem.py create mode 100644 scripts/check_remaining_agent_problems.py create mode 100644 scripts/check_two_events.py create mode 100644 scripts/check_zabbix_cluster.py create mode 100644 scripts/diagnose_zabbix.py create mode 100644 scripts/fix_swm01_output.py create mode 100644 scripts/inspect_assets_conf.py create mode 100644 scripts/persist_zabbix_nat.py create mode 100644 scripts/test_agent_dns.py create mode 100644 scripts/test_from_ctl01.py create mode 100644 scripts/test_zabbix_nat.py diff --git a/scripts/add_forward_rules.py b/scripts/add_forward_rules.py new file mode 100644 index 0000000..b6cd7c8 --- /dev/null +++ b/scripts/add_forward_rules.py @@ -0,0 +1,22 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + +run("iptables -I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT") +run("iptables -I FORWARD 2 -p tcp -d 140.44.4.64 --dport 10051 -j ACCEPT") +run("iptables -L FORWARD -n -v | head -n 8") + +ssh.close() + diff --git a/scripts/add_media_to_snarf.py b/scripts/add_media_to_snarf.py new file mode 100644 index 0000000..8964895 --- /dev/null +++ b/scripts/add_media_to_snarf.py @@ -0,0 +1,32 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php' + +payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'}) +token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +media_payload = { + 'jsonrpc': '2.0', + 'method': 'user.update', + 'params': { + 'userid': '3', + 'medias': [ + { + 'mediatypeid': '70', + 'sendto': 'snarf@snarfnet.net', + 'active': 0, + 'severity': 63, # 1+2+4+8+16+32 + 'period': '1-7,00:00-24:00' + } + ] + }, + 'id': 2 +} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(media_payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}) +res = json.loads(urllib.request.urlopen(req, context=ctx).read()) +print("User update result:", res) diff --git a/scripts/apply_h21_authentik.py b/scripts/apply_h21_authentik.py index d425763..f758a5f 100644 --- a/scripts/apply_h21_authentik.py +++ b/scripts/apply_h21_authentik.py @@ -34,6 +34,8 @@ location = /topology.html { error_page 401 = @goauthentik_proxy_signin; auth_request_set $auth_cookie $upstream_http_set_cookie; add_header Set-Cookie $auth_cookie; + add_header Cache-Control "no-cache, no-store, must-revalidate"; + expires 0; alias /data/topology/snarfnet_homelab_topology.html; default_type text/html; @@ -45,6 +47,8 @@ location = /topology.png { error_page 401 = @goauthentik_proxy_signin; auth_request_set $auth_cookie $upstream_http_set_cookie; add_header Set-Cookie $auth_cookie; + add_header Cache-Control "no-cache, no-store, must-revalidate"; + expires 0; alias /data/topology/snarfnet_homelab_topology.png; default_type image/png; @@ -55,6 +59,8 @@ location = /snarfnet_homelab_topology.html { error_page 401 = @goauthentik_proxy_signin; auth_request_set $auth_cookie $upstream_http_set_cookie; add_header Set-Cookie $auth_cookie; + add_header Cache-Control "no-cache, no-store, must-revalidate"; + expires 0; alias /data/topology/snarfnet_homelab_topology.html; default_type text/html; @@ -65,13 +71,14 @@ location = /snarfnet_homelab_topology.png { error_page 401 = @goauthentik_proxy_signin; auth_request_set $auth_cookie $upstream_http_set_cookie; add_header Set-Cookie $auth_cookie; + add_header Cache-Control "no-cache, no-store, must-revalidate"; + expires 0; alias /data/topology/snarfnet_homelab_topology.png; default_type image/png; } """ -# Upload adv_config to a file on remote stdin, stdout, stderr = c.exec_command("cat > /tmp/adv_config_h21.txt") stdin.write(adv_config) stdin.close() @@ -85,11 +92,7 @@ conf_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/ with open('/tmp/adv_config_h21.txt', 'r') as f: adv = f.read() -# 1. Backup -shutil.copy(db_path, db_path + '.bak_authentik') -shutil.copy(conf_path, conf_path + '.bak_authentik') - -# 2. Update DB +# Update DB conn = sqlite3.connect(db_path) cur = conn.cursor() cur.execute("UPDATE proxy_host SET advanced_config = ? WHERE id = 21", (adv,)) @@ -97,21 +100,29 @@ conn.commit() conn.close() print("Updated database for host 21.") -# 3. Read conf, replace or insert adv +# Update 21.conf with open(conf_path, 'r') as f: conf = f.read() -if '# Buffer tuning for Authentik headers' not in conf: +# Strip existing adv_config block if present +tag = '# Buffer tuning for Authentik headers' +if tag in conf: + idx_start = conf.find(tag) + idx_end = conf.find('location / {', idx_start) + if idx_start != -1 and idx_end != -1: + new_conf = conf[:idx_start] + adv + "\\n\\n " + conf[idx_end:] + else: + new_conf = conf +else: idx = conf.find('location / {') if idx != -1: new_conf = conf[:idx] + adv + "\\n\\n " + conf[idx:] else: new_conf = conf.rstrip().rstrip('}') + "\\n\\n" + adv + "\\n}\\n" - with open(conf_path, 'w') as f: - f.write(new_conf) - print("Updated 21.conf.") -else: - print("21.conf already has Authentik configuration.") + +with open(conf_path, 'w') as f: + f.write(new_conf) +print("Updated 21.conf.") """ stdin, stdout, stderr = c.exec_command("cat > /tmp/apply_h21.py") @@ -123,7 +134,6 @@ stdin, stdout, stderr = c.exec_command("python3 /tmp/apply_h21.py") print("Remote python stdout:", stdout.read().decode()) print("Remote python stderr:", stderr.read().decode()) -# Test nginx config inside container stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -t") out = stdout.read().decode() err = stderr.read().decode() @@ -135,9 +145,5 @@ if "successful" in out or "successful" in err: stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload") print("Reload stdout:", stdout.read().decode()) print("Reload stderr:", stderr.read().decode()) -else: - print("FAILED nginx -t test! Restoring backups...") - c.exec_command("cp /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/21.conf.bak_authentik /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/21.conf") - c.exec_command("cp /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite.bak_authentik /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite") c.close() diff --git a/scripts/audit_zabbix_alerts.py b/scripts/audit_zabbix_alerts.py new file mode 100644 index 0000000..2a3a01c --- /dev/null +++ b/scripts/audit_zabbix_alerts.py @@ -0,0 +1,48 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php" + +def zbx_req(method, params, token=None): + payload = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1} + headers = {'Content-Type': 'application/json-rpc'} + if token: + headers['Authorization'] = f'Bearer {token}' + req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method='POST') + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + return json.loads(r.read().decode())['result'] + +token = zbx_req('user.login', {'username': 'Admin', 'password': 'zabbix'}) +print("Zabbix Auth Token:", token) + +# 1. Actions +actions = zbx_req('action.get', {'output': 'extend', 'selectOperations': 'extend', 'selectFilter': 'extend'}, token) +print("\n=== TRIGGER ACTIONS ===") +for a in actions: + print(f"ID: {a['actionid']}, Name: {a['name']}, Status (0=enabled, 1=disabled): {a['status']}, EventSource: {a['eventsource']}") + print(f" Operations: {a.get('operations')}") + print(f" Filter: {a.get('filter')}") + +# 2. Media Types +media_types = zbx_req('mediatype.get', {'output': 'extend'}, token) +print("\n=== MEDIA TYPES ===") +for m in media_types: + print(f"ID: {m['mediatypeid']}, Name: {m['name']}, Type: {m['type']}, Status: {m['status']}") + +# 3. User Media +users = zbx_req('user.get', {'output': ['userid', 'username'], 'selectMedias': 'extend'}, token) +print("\n=== USERS & MEDIA ===") +for u in users: + print(f"User: {u['username']} (ID: {u['userid']})") + for med in u.get('medias', []): + print(f" Media: mediatypeid={med['mediatypeid']}, sendto={med['sendto']}, active={med['active']}, severity={med['severity']}") + +# 4. Recent Alerts +alerts = zbx_req('alert.get', {'output': 'extend', 'limit': 15, 'sortfield': 'clock', 'sortorder': 'DESC'}, token) +print("\n=== RECENT ALERTS ===") +for al in alerts: + print(f"Alert ID: {al['alertid']}, ActionID: {al['actionid']}, Clock: {al['clock']}, Error: {al.get('error')}, Status: {al.get('status')}, Subject: {al.get('subject')}") + diff --git a/scripts/cat_21_conf.py b/scripts/cat_21_conf.py index 6333b07..f2d470b 100644 --- a/scripts/cat_21_conf.py +++ b/scripts/cat_21_conf.py @@ -7,3 +7,4 @@ cmd = "docker exec $(docker ps -q --filter name=nginx-proxy-manager_app) cat /da _, o, _ = c.exec_command(cmd) print(o.read().decode()) c.close() + diff --git a/scripts/check_alerts_status.py b/scripts/check_alerts_status.py new file mode 100644 index 0000000..5276440 --- /dev/null +++ b/scripts/check_alerts_status.py @@ -0,0 +1,19 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php' + +payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'}) +token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +payload = {'jsonrpc': '2.0', 'method': 'alert.get', 'params': {'output': 'extend', 'limit': 20, 'sortfield': 'clock', 'sortorder': 'DESC'}, 'id': 2} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}) +alerts = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +for a in alerts: + print(f"ID: {a['alertid']} | Status: {a['status']} | User: {a['userid']} | Error: '{a.get('error')}' | Subject: {a.get('subject')}") + diff --git a/scripts/check_authentik_proxy_providers.py b/scripts/check_authentik_proxy_providers.py index fc33dcb..91cc772 100644 --- a/scripts/check_authentik_proxy_providers.py +++ b/scripts/check_authentik_proxy_providers.py @@ -22,3 +22,4 @@ print("\n=== Authentik Outposts ===") outposts = req("outposts/instances/") for o in outposts.get("results", []): print(o["pk"], o["name"], o["type"], [p for p in o.get("providers", [])]) + diff --git a/scripts/check_homepage_html.py b/scripts/check_homepage_html.py index c4f7793..f3420d7 100644 --- a/scripts/check_homepage_html.py +++ b/scripts/check_homepage_html.py @@ -13,3 +13,4 @@ for pat in [r'href="([^"]+)"', r'src="([^"]+)"']: for m in matches: if any(k in m for k in ['argo', 'tempest', 'weather', 'icon', 'png', 'svg']): print(m) + diff --git a/scripts/check_nat.py b/scripts/check_nat.py new file mode 100644 index 0000000..d599444 --- /dev/null +++ b/scripts/check_nat.py @@ -0,0 +1,20 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + print(o.read().decode().strip()) + err = e.read().decode().strip() + if err: + print("[stderr]", err) + +run("iptables -t nat -L PREROUTING -n -v") +run("iptables -t nat -L POSTROUTING -n -v") +run("sysctl net.ipv4.ip_forward") + +ssh.close() + diff --git a/scripts/check_npm_ports.py b/scripts/check_npm_ports.py new file mode 100644 index 0000000..f6bbfe5 --- /dev/null +++ b/scripts/check_npm_ports.py @@ -0,0 +1,20 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +stdin, stdout, stderr = ssh.exec_command("docker ps --filter 'name=nginx-proxy-manager_app' --format '{{.Ports}}'") +print("NPM Container Ports:") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command("docker inspect $(docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{json .HostConfig.PortBindings}}'") +print("NPM HostConfig PortBindings:") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command("docker inspect $(docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{.HostConfig.NetworkMode}}'") +print("NPM NetworkMode:") +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/check_npm_streams.py b/scripts/check_npm_streams.py new file mode 100644 index 0000000..e06fb1b --- /dev/null +++ b/scripts/check_npm_streams.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +stdin, stdout, stderr = ssh.exec_command("python3 -c \"import sqlite3; con = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'); print(list(con.execute('SELECT id, incoming_port, forwarding_host, forwarding_port, is_deleted FROM stream')))\"") +print("STREAMS:", stdout.read().decode()) +print("ERR:", stderr.read().decode()) + +ssh.close() diff --git a/scripts/check_one_problem.py b/scripts/check_one_problem.py new file mode 100644 index 0000000..4d0bb40 --- /dev/null +++ b/scripts/check_one_problem.py @@ -0,0 +1,24 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php' + +payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'}) +token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +payload = {'jsonrpc': '2.0', 'method': 'problem.get', 'params': {'output': 'extend'}, 'id': 2} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}) +problems = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +for p in problems: + if 'active checks' in p['name'].lower(): + # Get event + ep = {'jsonrpc': '2.0', 'method': 'event.get', 'params': {'eventids': [p['eventid']], 'selectHosts': ['host', 'name']}, 'id': 3} + er = urllib.request.Request(ZABBIX_URL, data=json.dumps(ep).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}) + events = json.loads(urllib.request.urlopen(er, context=ctx).read())['result'] + print(f"Problem: {p['name']} | Event: {events}") + diff --git a/scripts/check_remaining_agent_problems.py b/scripts/check_remaining_agent_problems.py new file mode 100644 index 0000000..d177e81 --- /dev/null +++ b/scripts/check_remaining_agent_problems.py @@ -0,0 +1,23 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php' + +payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'}) +token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +payload = {'jsonrpc': '2.0', 'method': 'problem.get', 'params': {'output': 'extend', 'selectAcknowledges': 'extend'}, 'id': 2} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}) +resp = json.loads(urllib.request.urlopen(req, context=ctx).read()) +if 'error' in resp: + print("API Error:", resp['error']) +else: + problems = resp['result'] + print(f"Total problems: {len(problems)}") + for p in problems: + if 'agent' in p['name'].lower(): + print(f" - EventID: {p['eventid']} | Severity: {p['severity']} | Name: {p['name']}") diff --git a/scripts/check_two_events.py b/scripts/check_two_events.py new file mode 100644 index 0000000..1344247 --- /dev/null +++ b/scripts/check_two_events.py @@ -0,0 +1,19 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php' + +payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'}) +token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +payload = {'jsonrpc': '2.0', 'method': 'event.get', 'params': {'eventids': ['82136', '82113'], 'selectHosts': ['host', 'name']}, 'id': 2} +req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}) +events = json.loads(urllib.request.urlopen(req, context=ctx).read())['result'] + +for e in events: + print(f"Event {e['eventid']}: name={e['name']}, hosts={e.get('hosts')}") + diff --git a/scripts/check_zabbix_cluster.py b/scripts/check_zabbix_cluster.py new file mode 100644 index 0000000..3842c08 --- /dev/null +++ b/scripts/check_zabbix_cluster.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +# Let's check Spaceport master 140.44.4.41 for CoreDNS / Kubernetes or Pi-hole +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + +run("kubectl get svc -n monitoring") +run("kubectl get pods -n monitoring") +run("kubectl logs deployment/zabbix-zabbix-server -n monitoring -c zabbix-server --tail=50") + +ssh.close() + diff --git a/scripts/copy_topology_to_npm.py b/scripts/copy_topology_to_npm.py index 917bf35..ab24d2c 100644 --- a/scripts/copy_topology_to_npm.py +++ b/scripts/copy_topology_to_npm.py @@ -1,35 +1,32 @@ -import paramiko +import paramiko, os c = paramiko.SSHClient() c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) c.connect("140.44.4.71", username="root", key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) -# Create /data/topology directory on host -c.exec_command("mkdir -p /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/topology") +sftp = c.open_sftp() -# Read topology files locally -with open('docs/snarfnet_homelab_topology.html', 'r', encoding='utf-8') as f: - html_data = f.read() +remote_dir = "/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/topology" +c.exec_command(f"mkdir -p {remote_dir}") -with open('docs/snarfnet_homelab_topology.png', 'rb') as f: - png_data = f.read() +local_html = os.path.abspath("docs/snarfnet_homelab_topology.html") +local_png = os.path.abspath("docs/snarfnet_homelab_topology.png") -# Send html -stdin, stdout, stderr = c.exec_command("cat > /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/topology/snarfnet_homelab_topology.html") -stdin.write(html_data) -stdin.close() -stdout.read() +print(f"Uploading HTML ({os.path.getsize(local_html)} bytes)...") +sftp.put(local_html, f"{remote_dir}/snarfnet_homelab_topology.html") -# Send png -stdin, stdout, stderr = c.exec_command("cat > /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/topology/snarfnet_homelab_topology.png") -stdin.write(png_data) -stdin.close() -stdout.read() +print(f"Uploading PNG ({os.path.getsize(local_png)} bytes)...") +sftp.put(local_png, f"{remote_dir}/snarfnet_homelab_topology.png") -# Verify files inside NPM container -cmd = "docker exec $(docker ps -q --filter name=nginx-proxy-manager_app) ls -la /data/topology" -_, o, e = c.exec_command(cmd) -print("Files inside NPM container at /data/topology:\n" + o.read().decode()) -print("Stderr:\n" + e.read().decode()) +sftp.close() + +# Verify files inside container +stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter name=nginx-proxy-manager_app) ls -la /data/topology") +print("Container /data/topology:") +print(stdout.read().decode()) + +# Check md5sum locally and remotely +stdin, stdout, stderr = c.exec_command(f"md5sum {remote_dir}/*") +print("Remote md5sum:\n", stdout.read().decode()) c.close() diff --git a/scripts/diagnose_zabbix.py b/scripts/diagnose_zabbix.py new file mode 100644 index 0000000..5050712 --- /dev/null +++ b/scripts/diagnose_zabbix.py @@ -0,0 +1,22 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + +run("systemctl status zabbix-agent2") +run("grep -E '^Server|^Hostname|^HostMetadata' /etc/zabbix/zabbix_agent2.conf") +run("tail -n 25 /var/log/zabbix/zabbix_agent2.log") + +ssh.close() + diff --git a/scripts/fix_swm01_output.py b/scripts/fix_swm01_output.py new file mode 100644 index 0000000..6384569 --- /dev/null +++ b/scripts/fix_swm01_output.py @@ -0,0 +1,24 @@ +import paramiko, time + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + +run("iptables -t nat -I OUTPUT 1 -p tcp -d 140.44.4.70 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051") +run("iptables -t nat -I OUTPUT 2 -p tcp -d 140.44.4.71 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051") +run("systemctl restart zabbix-agent2") +time.sleep(3) +run("tail -n 15 /var/log/zabbix/zabbix_agent2.log") + +ssh.close() + diff --git a/scripts/inspect_assets_conf.py b/scripts/inspect_assets_conf.py new file mode 100644 index 0000000..6bce0d4 --- /dev/null +++ b/scripts/inspect_assets_conf.py @@ -0,0 +1,13 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = "docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') cat /etc/nginx/conf.d/include/assets.conf" +stdin, stdout, stderr = ssh.exec_command(cmd) +print("--- assets.conf ---") +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_npm_adv.py b/scripts/inspect_npm_adv.py index 57cc71c..2f45850 100644 --- a/scripts/inspect_npm_adv.py +++ b/scripts/inspect_npm_adv.py @@ -23,3 +23,4 @@ stdout.read() stdin, stdout, stderr = c.exec_command("python3 /tmp/check_adv.py") print(stdout.read().decode()) c.close() + diff --git a/scripts/inspect_npm_container.py b/scripts/inspect_npm_container.py index 14d7d75..9d19285 100644 --- a/scripts/inspect_npm_container.py +++ b/scripts/inspect_npm_container.py @@ -12,3 +12,4 @@ _, o, _ = c.exec_command(cmd) print("Mounts:\n" + o.read().decode()) c.close() + diff --git a/scripts/persist_zabbix_nat.py b/scripts/persist_zabbix_nat.py new file mode 100644 index 0000000..87c9229 --- /dev/null +++ b/scripts/persist_zabbix_nat.py @@ -0,0 +1,39 @@ +import paramiko + +def apply_to_host(ip): + print(f"=== Applying to {ip} ===") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + commands = [ + "iptables -t nat -C PREROUTING -p tcp --dport 10051 -j DNAT --to-destination 140.44.4.64:10051 2>/dev/null || iptables -t nat -I PREROUTING 1 -p tcp --dport 10051 -j DNAT --to-destination 140.44.4.64:10051", + "iptables -t nat -C POSTROUTING -p tcp -d 140.44.4.64 --dport 10051 -j MASQUERADE 2>/dev/null || iptables -t nat -I POSTROUTING 1 -p tcp -d 140.44.4.64 --dport 10051 -j MASQUERADE", + "iptables -t nat -C OUTPUT -p tcp -d 140.44.4.70 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051 2>/dev/null || iptables -t nat -I OUTPUT 1 -p tcp -d 140.44.4.70 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051", + "iptables -t nat -C OUTPUT -p tcp -d 140.44.4.71 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051 2>/dev/null || iptables -t nat -I OUTPUT 2 -p tcp -d 140.44.4.71 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051", + "iptables -t nat -C OUTPUT -p tcp -d 140.44.4.72 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051 2>/dev/null || iptables -t nat -I OUTPUT 3 -p tcp -d 140.44.4.72 --dport 10051 -j DNAT --to-destination 140.44.4.64:10051", + "iptables -C FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || iptables -I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT", + "iptables -C FORWARD -p tcp -d 140.44.4.64 --dport 10051 -j ACCEPT 2>/dev/null || iptables -I FORWARD 2 -p tcp -d 140.44.4.64 --dport 10051 -j ACCEPT", + "sysctl -w net.ipv4.ip_forward=1", + "which iptables-save" + ] + + for cmd in commands: + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + if out: + print(f" {out}") + err = e.read().decode().strip() + if err: + print(f" [err] {err}") + + # Check persistence package (iptables-persistent or netfilter-persistent) + stdin, stdout, stderr = ssh.exec_command("which netfilter-persistent || which iptables-save") + print(f" Persistence tool: {stdout.read().decode().strip()}") + stdin, stdout, stderr = ssh.exec_command("iptables-save > /etc/iptables/rules.v4 2>/dev/null || iptables-save > /etc/iptables.rules 2>/dev/null || true") + + ssh.close() + +apply_to_host("140.44.4.71") +apply_to_host("140.44.4.72") + diff --git a/scripts/test_agent_dns.py b/scripts/test_agent_dns.py new file mode 100644 index 0000000..d72dc87 --- /dev/null +++ b/scripts/test_agent_dns.py @@ -0,0 +1,21 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + print(o.read().decode().strip()) + err = e.read().decode().strip() + if err: + print("[stderr]", err) + +run("cat /etc/resolv.conf") +run("getent hosts zabbix.snarfnet.net") +run("nc -zv 140.44.4.64 10051") +run("nc -zv zabbix.snarfnet.net 10051") + +ssh.close() + diff --git a/scripts/test_from_ctl01.py b/scripts/test_from_ctl01.py new file mode 100644 index 0000000..670c0c6 --- /dev/null +++ b/scripts/test_from_ctl01.py @@ -0,0 +1,21 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + +run("nc -zv zabbix.snarfnet.net 10051") +run("tail -n 20 /var/log/zabbix/zabbix_agent2.log") + +ssh.close() + diff --git a/scripts/test_zabbix_nat.py b/scripts/test_zabbix_nat.py new file mode 100644 index 0000000..148d380 --- /dev/null +++ b/scripts/test_zabbix_nat.py @@ -0,0 +1,22 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f">>> {cmd}") + _, o, e = ssh.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + +run("iptables -t nat -I PREROUTING 1 -p tcp --dport 10051 -j DNAT --to-destination 140.44.4.64:10051") +run("iptables -t nat -I POSTROUTING 1 -p tcp -d 140.44.4.64 --dport 10051 -j MASQUERADE") +run("iptables -t nat -L PREROUTING -n -v | head -n 5") + +ssh.close() +