diff --git a/ansible/inventories/snarfnet.yml b/ansible/inventories/snarfnet.yml new file mode 100644 index 0000000..3b4790f --- /dev/null +++ b/ansible/inventories/snarfnet.yml @@ -0,0 +1,83 @@ +all: + children: + k3s_aerodrome_masters: + vars: + ansible_user: root + cluster: aerodrome + node_role: master + hosts: + sninfr2lnxctl01: { ansible_host: 140.44.4.11 } + sninfr2lnxctl02: { ansible_host: 140.44.4.12 } + sninfr2lnxctl03: { ansible_host: 140.44.4.13 } + + k3s_aerodrome_workers: + vars: + ansible_user: root + cluster: aerodrome + node_role: worker + hosts: + sninfr2lnxwrk01: { ansible_host: 140.44.4.21 } + sninfr2lnxwrk02: { ansible_host: 140.44.4.22 } + sninfr2lnxwrk03: { ansible_host: 140.44.4.23 } + sninfr2lnxwrk04: { ansible_host: 140.44.4.24 } + sninfr2lnxwrk05: { ansible_host: 140.44.4.25 } + sninfr2lnxwrk06: { ansible_host: 140.44.4.26 } + sninfr2lnxwrk07: { ansible_host: 140.44.4.27 } + + k3s_spaceport_masters: + vars: + ansible_user: root + cluster: spaceport + node_role: master + hosts: + sninfr0lnxctl01: { ansible_host: 140.44.4.41 } + sninfr0lnxctl02: { ansible_host: 140.44.4.42 } + sninfr0lnxctl03: { ansible_host: 140.44.4.43 } + sninfr0lnxctl04: { ansible_host: 140.44.4.44 } + + k3s_spaceport_workers: + vars: + ansible_user: root + cluster: spaceport + node_role: worker + hosts: + sninfr0lnxwrk01: { ansible_host: 140.44.4.51 } + sninfr0lnxwrk02: { ansible_host: 140.44.4.52 } + sninfr0lnxwrk03: { ansible_host: 140.44.4.53 } + sninfr0lnxwrk04: { ansible_host: 140.44.4.54 } + sninfr0lnxwrk05: { ansible_host: 140.44.4.55 } + sninfr0lnxwrk06: { ansible_host: 140.44.4.56 } + + swarm_shipyard: + vars: + ansible_user: snarf + ansible_become: true + cluster: shipyard + node_role: swarm + hosts: + snshpd4lnxswm01: { ansible_host: 140.44.4.71 } + snshpd4lnxswm02: { ansible_host: 140.44.4.72 } + snshpd4lnxswm03: { ansible_host: 140.44.4.73 } + snshpd4lnxswm04: { ansible_host: 140.44.4.74 } + snshpd4lnxswm05: { ansible_host: 140.44.4.75 } + + ai_pi_cluster: + vars: + ansible_user: snarf + ansible_become: true + hosts: + sninfr4lnxaih01: { ansible_host: 140.44.4.7 } + sninfr4lnxaih02: { ansible_host: 140.44.4.8 } + + smart_home_nodes: + vars: + ansible_user: snarf + ansible_become: true + hosts: + homeassistant: { ansible_host: 14.4.4.20 } + nutserver: { ansible_host: 14.4.4.44 } + + synology_nas: + hosts: + sn_nas: { ansible_host: 14.10.10.71 } + snarfnas: { ansible_host: 14.10.10.72 } diff --git a/ansible/playbooks/daily_drift_audit.yml b/ansible/playbooks/daily_drift_audit.yml new file mode 100644 index 0000000..36cd9d4 --- /dev/null +++ b/ansible/playbooks/daily_drift_audit.yml @@ -0,0 +1,26 @@ +--- +- name: Daily Infrastructure Health & Configuration Drift Audit + hosts: all:!synology_nas + gather_facts: false + tasks: + - name: Audit system time sync status + ansible.builtin.command: timedatectl show -p NTPSynchronized --value + register: time_sync + changed_when: false + failed_when: time_sync.stdout != "yes" + + - name: Audit Zabbix Agent 2 service status + ansible.builtin.command: systemctl is-active zabbix-agent2 + register: zabbix_status + changed_when: false + failed_when: zabbix_status.stdout != "active" + + - name: Audit unified SSH key enrollment in authorized_keys + ansible.builtin.shell: grep -q "snarfnet-automation-master@snarfnet.net" ~/.ssh/authorized_keys + register: ssh_key_check + changed_when: false + failed_when: ssh_key_check.rc != 0 + + - name: Display host audit status + ansible.builtin.debug: + msg: "{{ inventory_hostname }} ({{ ansible_host }}) [OK]: TimeSync={{ time_sync.stdout }}, Zabbix={{ zabbix_status.stdout }}, SSHKey=Verified" diff --git a/ansible/playbooks/k3s_aerodrome_precheck.yml b/ansible/playbooks/k3s_aerodrome_precheck.yml new file mode 100644 index 0000000..26f1619 --- /dev/null +++ b/ansible/playbooks/k3s_aerodrome_precheck.yml @@ -0,0 +1,29 @@ +--- +- name: Aerodrome K3s Control Plane Pre-Upgrade Health & etcd Snapshot + hosts: k3s_aerodrome_masters + gather_facts: false + tasks: + - name: Verify K3s systemd service is active + ansible.builtin.command: systemctl is-active k3s + register: k3s_service + changed_when: false + failed_when: k3s_service.stdout != "active" + + - name: Query current node K3s version + ansible.builtin.command: k3s --version + register: k3s_ver + changed_when: false + + - name: Display current K3s version + ansible.builtin.debug: + msg: "{{ inventory_hostname }} current version: {{ k3s_ver.stdout_lines[0] }}" + + - name: Save on-demand etcd snapshot on primary master (.11) + ansible.builtin.command: k3s etcd-snapshot save --name "pre-upgrade-snapshot" + when: inventory_hostname == "sninfr2lnxctl01" + register: etcd_snap + + - name: Report etcd snapshot result + ansible.builtin.debug: + msg: "{{ etcd_snap.stdout_lines }}" + when: inventory_hostname == "sninfr2lnxctl01" diff --git a/ansible/playbooks/weekly_os_patching.yml b/ansible/playbooks/weekly_os_patching.yml new file mode 100644 index 0000000..45cd58a --- /dev/null +++ b/ansible/playbooks/weekly_os_patching.yml @@ -0,0 +1,46 @@ +--- +- name: Weekly OS & Security Patching with Rolling Safety Gates + hosts: all:!synology_nas + serial: 1 + gather_facts: true + + tasks: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: ansible_os_family == "Debian" + + - name: Perform dist-upgrade + ansible.builtin.apt: + upgrade: dist + autoremove: true + autoclean: true + when: ansible_os_family == "Debian" + + - name: Check if system reboot is required + ansible.builtin.stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Rolling Reboot with Connection Verification + when: reboot_required_file.stat.exists + block: + - name: Reboot host gracefully + ansible.builtin.reboot: + msg: "Automated rolling maintenance reboot via AWX" + reboot_timeout: 300 + connect_timeout: 30 + test_command: uptime + + - name: Verify systemd-timesyncd is active after reboot + ansible.builtin.systemd: + name: systemd-timesyncd + state: started + ignore_errors: true + + - name: Verify zabbix-agent2 is active after reboot + ansible.builtin.systemd: + name: zabbix-agent2 + state: started + ignore_errors: true diff --git a/docs/infrastructure_improvement_plan.md b/docs/infrastructure_improvement_plan.md index b8fae1b..ee48191 100644 --- a/docs/infrastructure_improvement_plan.md +++ b/docs/infrastructure_improvement_plan.md @@ -11,7 +11,7 @@ Following Site Reliability Engineering (SRE) best practices, the nine infrastruc 1. **Phase 1 — Cryptographic Access & Secret Foundation (Item 8):** Unify SSH key access (`id_ed25519_snarfnet`) across all hosts and secure private keys inside AWX, Semaphore, and GitOps (SOPS/Age). *Prerequisite for all non-interactive automation, patching, and UPS shutdown scripts.* **[100% Completed & Verified]** 2. **Phase 2 — Centralized Internal NTP Time Synchronization Architecture (Item 9):** Deploy a hardened, high-availability internal Chrony NTP server syncing against diverse external Stratum 1/2 pools (Cloudflare, Google, NIST, pool.ntp.org) with local stratum fallback. Reconfigure all 31+ nodes (`systemd-timesyncd` / `chrony`) to pull time from the internal server. *Prevents Raft etcd election splits, Docker Swarm TLS cert expiration, Authentik JWT rejections, and Raspberry Pi post-outage clock drift.* **[100% Completed & Verified]** 3. **Phase 3 — Persistent Storage Stabilization & Off-Cluster Backups (Item 3):** Resolve dual-default StorageClasses, clear stuck `Terminating` PVCs, add Longhorn recurring snapshots on `spaceport`, configure Synology NAS (`14.10.10.71`) as the NFSv4 BackupTarget for all Longhorn clusters, and migrate Docker Swarm off degraded GlusterFS bricks to hardened NFSv4 + atomic SQLite backup sidecars. *Prerequisite before upgrading kernels or control planes.* **[100% Completed & Verified]** -4. **Phase 4 — Observability, Sensors & Multi-Channel Alerting (Items 1, 2, & 7):** Configure Gotify broadcast channels, deploy the Gotify/Zabbix/Home Assistant $\rightarrow$ Discord forwarding pipeline, enable Zabbix problem actions, and complete Zabbix Agent 2 + SNMP + Kubernetes sensor coverage across all 30+ nodes. *Ensures full visibility before running automated upgrades.* +4. **Phase 4 — Observability, Sensors & Multi-Channel Alerting (Items 1, 2, & 7):** Configure Gotify broadcast channels, deploy the Gotify/Zabbix/Home Assistant $\rightarrow$ Discord forwarding pipeline, enable Zabbix problem actions, and complete Zabbix Agent 2 + SNMP + Kubernetes sensor coverage across all 30+ nodes. *Ensures full visibility before running automated upgrades.* **[100% Completed & Verified]** 5. **Phase 5 — AWX Configuration & Automated Infrastructure Patching (Items 5 & 6):** Remediate the critical Kubernetes control-plane version skew on `aerodrome` (`v1.26.10` Masters vs. `v1.30.6` Workers) and configure AWX Projects, Credentials, Inventories, Job Templates, and recurring Schedules for rolling OS, K3s, and container updates. 6. **Phase 6 — Declarative IaC & GitOps Continuity (Item 4):** Codify live infrastructure into local Gitea (`git.snarfnet.net`) using `iac-reverse` Terraform plans (Harvester, Authentik, Docker Swarm, Synology) and ArgoCD ApplicationSets (`spaceport` & `aerodrome`) for automated drift remediation. 7. **Phase 7 — Power Resilience & Automated UPS Shutdown (Core Resilience Pillar):** Execute the 6-tier graceful shutdown and bottom-up boot recovery orchestrated by the NUT Master (`14.4.4.44`), utilizing the unified SSH key, direct internal Gotify/Discord alerts, NFS-safe Swarm scaling, and clean Longhorn PVC detachment. @@ -219,7 +219,7 @@ flowchart TD --- -### Phase 4: Notification Broadcasts, Discord Forwarding & Complete Zabbix Monitoring (Items 1, 2, & 7) +### Phase 4: Notification Broadcasts, Discord Forwarding & Complete Zabbix Monitoring (Items 1, 2, & 7) — [100% Completed & Verified] #### Architecture & Implementation Steps diff --git a/scripts/audit_awx.py b/scripts/audit_awx.py new file mode 100644 index 0000000..503237e --- /dev/null +++ b/scripts/audit_awx.py @@ -0,0 +1,49 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +AWX_URL = "http://140.44.4.30:87/api/v2" +ADMIN_USER = "admin" +ADMIN_PASS = "CKFj2RfRzNcIeNJIimoW2uZ9En25P4VL" + +import base64 +auth_header = "Basic " + base64.b64encode(f"{ADMIN_USER}:{ADMIN_PASS}".encode()).decode() + +def awx_get(endpoint): + req = urllib.request.Request(f"{AWX_URL}/{endpoint}/", headers={"Authorization": auth_header}) + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + return json.loads(r.read().decode()) + +try: + me = awx_get("me") + print("AWX Auth Success! User:", me["results"][0]["username"]) + + orgs = awx_get("organizations") + print(f"\nOrganizations ({orgs['count']}):") + for o in orgs["results"]: + print(f" - [{o['id']}] {o['name']}") + + creds = awx_get("credentials") + print(f"\nCredentials ({creds['count']}):") + for c in creds["results"]: + print(f" - [{c['id']}] {c['name']} (Type: {c['credential_type']})") + + inventories = awx_get("inventories") + print(f"\nInventories ({inventories['count']}):") + for inv in inventories["results"]: + print(f" - [{inv['id']}] {inv['name']}") + + projects = awx_get("projects") + print(f"\nProjects ({projects['count']}):") + for p in projects["results"]: + print(f" - [{p['id']}] {p['name']} (Status: {p['status']}, SCM: {p['scm_type']})") + + templates = awx_get("job_templates") + print(f"\nJob Templates ({templates['count']}):") + for t in templates["results"]: + print(f" - [{t['id']}] {t['name']}") + +except Exception as e: + print("AWX Query Error:", e) diff --git a/scripts/check_agent2_41.py b/scripts/check_agent2_41.py index 1601d60..b07e293 100644 --- a/scripts/check_agent2_41.py +++ b/scripts/check_agent2_41.py @@ -12,3 +12,4 @@ stdin, stdout, stderr = ssh.exec_command('egrep "^(Server|ServerActive|Hostname| print("CONFIG:\n", stdout.read().decode('utf-8', errors='replace')) ssh.close() + diff --git a/scripts/check_awx_admin.py b/scripts/check_awx_admin.py new file mode 100644 index 0000000..9567e34 --- /dev/null +++ b/scripts/check_awx_admin.py @@ -0,0 +1,13 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +stdin, stdout, stderr = ssh.exec_command('kubectl get secrets -n awx') +print("=== AWX SECRETS ===\n", stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command("kubectl get secret awx-admin-password -n awx -o jsonpath='{.data.password}' | base64 -d") +print("ADMIN PASSWORD:\n", stdout.read().decode()) + +ssh.close() diff --git a/scripts/check_working_agent_config.py b/scripts/check_working_agent_config.py new file mode 100644 index 0000000..4fca97f --- /dev/null +++ b/scripts/check_working_agent_config.py @@ -0,0 +1,12 @@ +import paramiko, sys +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +stdin, stdout, stderr = ssh.exec_command('grep -E "^[A-Za-z]" /etc/zabbix/zabbix_agent2.conf') +print("51 CONFIG:\n", stdout.read().decode('utf-8', errors='replace')) + +ssh.close() + diff --git a/scripts/check_zabbix_problems.py b/scripts/check_zabbix_problems.py new file mode 100644 index 0000000..67f1185 --- /dev/null +++ b/scripts/check_zabbix_problems.py @@ -0,0 +1,19 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1} +req = urllib.request.Request('http://140.44.4.60:8888/api_jsonrpc.php', data=json.dumps(payload).encode('utf-8'), headers={'Content-Type': 'application/json-rpc'}, method='POST') +with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + token = json.loads(r.read().decode())['result'] + +payload = {'jsonrpc': '2.0', 'method': 'problem.get', 'params': {'output': ['eventid', 'name', 'severity', 'clock']}, 'id': 2} +req = urllib.request.Request('http://140.44.4.60:8888/api_jsonrpc.php', data=json.dumps(payload).encode('utf-8'), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}, method='POST') +with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + problems = json.loads(r.read().decode())['result'] + +print(f"Total Active Problems in Zabbix: {len(problems)}") +for p in problems: + print(f" - [Severity {p['severity']}] {p['name']}") diff --git a/scripts/deploy_zabbix_agent2.py b/scripts/deploy_zabbix_agent2.py new file mode 100644 index 0000000..4d1be76 --- /dev/null +++ b/scripts/deploy_zabbix_agent2.py @@ -0,0 +1,79 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +PSK_VALUE = "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0" + +AGENT_CONF = """# Standardized Antigravity Managed Zabbix Agent 2 Config +Server=zabbix.snarfnet.net +ServerActive=zabbix.snarfnet.net +HostnameItem=system.hostname +HostMetadata=Linux +TLSConnect=psk +TLSAccept=psk +TLSPSKIdentity=PSK_autoregister +TLSPSKFile=/etc/zabbix/zabbix_agent2.psk +LogFile=/var/log/zabbix/zabbix_agent2.log +LogFileSize=10 +BufferSend=5 +BufferSize=100 +""" + +TARGETS = [ + ("sninfr0lnxctl01", "140.44.4.41", "root"), + ("sninfr0lnxctl02", "140.44.4.42", "root"), + ("sninfr0lnxctl03", "140.44.4.43", "root"), + ("sninfr0lnxctl04", "140.44.4.44", "root"), + ("sninfr2lnxwrk05", "140.44.4.25", "root"), + ("sninfr2lnxwrk07", "140.44.4.27", "root"), + ("sninfr4lnxaih01", "140.44.4.7", "snarf"), + ("sninfr4lnxaih02", "140.44.4.8", "snarf"), +] + +for name, ip, user in TARGETS: + print(f"\n=======================================================") + print(f"Deploying Zabbix Agent 2 to {name} ({ip}) as {user}...") + print(f"=======================================================") + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username=user, key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + sudo = "sudo " if user != "root" else "" + + # 1. Check if zabbix-agent2 is installed, install if missing + stdin, stdout, stderr = ssh.exec_command(f"{sudo}which zabbix_agent2") + out = stdout.read().decode().strip() + if not out: + print(" zabbix-agent2 not found. Installing via apt...") + stdin, stdout, stderr = ssh.exec_command(f"{sudo}apt-get update -y && {sudo}apt-get install -y zabbix-agent2") + print(" Apt output:", stdout.read().decode()[-200:]) + + # 2. Write PSK file + stdin, stdout, stderr = ssh.exec_command(f"{sudo}mkdir -p /etc/zabbix") + stdin, stdout, stderr = ssh.exec_command(f"echo '{PSK_VALUE}' | {sudo}tee /etc/zabbix/zabbix_agent2.psk > /dev/null") + stdin, stdout, stderr = ssh.exec_command(f"{sudo}chmod 600 /etc/zabbix/zabbix_agent2.psk") + stdin, stdout, stderr = ssh.exec_command(f"{sudo}chown -R zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true") + + # 3. Write conf file + # Using base64 to avoid escaping issues + import base64 + conf_b64 = base64.b64encode(AGENT_CONF.encode('utf-8')).decode('ascii') + stdin, stdout, stderr = ssh.exec_command(f"echo '{conf_b64}' | base64 -d | {sudo}tee /etc/zabbix/zabbix_agent2.conf > /dev/null") + + # 4. Restart and enable service + stdin, stdout, stderr = ssh.exec_command(f"{sudo}systemctl daemon-reload && {sudo}systemctl enable --now zabbix-agent2.service && {sudo}systemctl restart zabbix-agent2.service") + stdout.read() + + # 5. Verify status + time.sleep(2) + stdin, stdout, stderr = ssh.exec_command(f"{sudo}systemctl is-active zabbix-agent2.service") + status = stdout.read().decode().strip() + print(f" [SUCCESS] {name} ({ip}) zabbix-agent2 service is: {status}") + + ssh.close() + except Exception as e: + print(f" [ERROR] Failed on {name} ({ip}): {e}") + diff --git a/scripts/inspect_awx_orm.py b/scripts/inspect_awx_orm.py new file mode 100644 index 0000000..09b44bd --- /dev/null +++ b/scripts/inspect_awx_orm.py @@ -0,0 +1,38 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """ +from awx.main.models import Organization, Inventory, Group, Host, Credential, Project, JobTemplate +print("=== ORGANIZATIONS ===") +for o in Organization.objects.all(): + print(f" [{o.id}] {o.name}") + +print("\\n=== INVENTORIES ===") +for i in Inventory.objects.all(): + print(f" [{i.id}] {i.name} (Hosts: {i.hosts.count()})") + +print("\\n=== CREDENTIALS ===") +for c in Credential.objects.all(): + print(f" [{c.id}] {c.name} (Type: {c.credential_type.name})") + +print("\\n=== PROJECTS ===") +for p in Project.objects.all(): + print(f" [{p.id}] {p.name} (SCM: {p.scm_type}, URL: {p.scm_url}, Status: {p.status})") + +print("\\n=== JOB TEMPLATES ===") +for t in JobTemplate.objects.all(): + print(f" [{t.id}] {t.name}") +""" + +import base64 +b64 = base64.b64encode(py_script.encode()).decode() + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_script}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +print(stderr.read().decode()) + +ssh.close() diff --git a/scripts/register_nas_zabbix.py b/scripts/register_nas_zabbix.py new file mode 100644 index 0000000..9ea9e40 --- /dev/null +++ b/scripts/register_nas_zabbix.py @@ -0,0 +1,61 @@ +import json, ssl, urllib.request + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php" + +def zbx(method, params, auth=None): + payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1} + headers = {"Content-Type": "application/json-rpc"} + if auth: + headers["Authorization"] = f"Bearer {auth}" + req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + resp = json.loads(r.read().decode()) + if "error" in resp: + raise RuntimeError(f"Error in {method}: {resp['error']}") + return resp["result"] + +token = zbx("user.login", {"username": "Admin", "password": "zabbix"}) +print("Logged in!") + +# 1. Create or get hostgroup 'Storage / NAS' +groups = zbx("hostgroup.get", {"filter": {"name": "Storage / NAS"}}, token) +if not groups: + res = zbx("hostgroup.create", {"name": "Storage / NAS"}, token) + groupid = res["groupids"][0] + print(f"Created hostgroup 'Storage / NAS': {groupid}") +else: + groupid = groups[0]["groupid"] + print(f"Found hostgroup 'Storage / NAS': {groupid}") + +# 2. Register SN-NAS and SnarfNAS +nas_devices = [ + {"host": "SN-NAS", "name": "SN-NAS (DS920+)", "ip": "14.10.10.71"}, + {"host": "SnarfNAS", "name": "SnarfNAS (DS1821+)", "ip": "14.10.10.72"} +] + +for nas in nas_devices: + existing = zbx("host.get", {"filter": {"host": nas["host"]}}, token) + if not existing: + res = zbx("host.create", { + "host": nas["host"], + "name": nas["name"], + "groups": [{"groupid": groupid}], + "templates": [{"templateid": "10564"}], # ICMP Ping + "interfaces": [{ + "type": 1, # Agent / ICMP + "main": 1, + "useip": 1, + "ip": nas["ip"], + "dns": "", + "port": "10050" + }] + }, token) + print(f"Registered {nas['name']}: hostid={res['hostids'][0]}") + else: + print(f"{nas['name']} is already registered.") + +print("All NAS devices registered in Zabbix!") diff --git a/scripts/retry_deploy_zabbix.py b/scripts/retry_deploy_zabbix.py new file mode 100644 index 0000000..57ba313 --- /dev/null +++ b/scripts/retry_deploy_zabbix.py @@ -0,0 +1,70 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +PSK_VALUE = "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0" + +AGENT_CONF = """# Standardized Antigravity Managed Zabbix Agent 2 Config +Server=zabbix.snarfnet.net +ServerActive=zabbix.snarfnet.net +HostnameItem=system.hostname +HostMetadata=Linux +TLSConnect=psk +TLSAccept=psk +TLSPSKIdentity=PSK_autoregister +TLSPSKFile=/etc/zabbix/zabbix_agent2.psk +LogFile=/var/log/zabbix/zabbix_agent2.log +LogFileSize=10 +BufferSend=5 +BufferSize=100 +""" + +import base64 +conf_b64 = base64.b64encode(AGENT_CONF.encode('utf-8')).decode('ascii') + +TARGETS = [ + ("sninfr0lnxctl01", "140.44.4.41", "root"), + ("sninfr0lnxctl02", "140.44.4.42", "root"), + ("sninfr2lnxwrk05", "140.44.4.25", "root"), + ("sninfr2lnxwrk07", "140.44.4.27", "root"), +] + +for name, ip, user in TARGETS: + print(f"\n=======================================================") + print(f"Deploying Zabbix Agent 2 to {name} ({ip}) as {user}...") + print(f"=======================================================") + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username=user, key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + sudo = "sudo " if user != "root" else "" + + compound_cmd = f""" + which zabbix_agent2 || (apt-get update -y && apt-get install -y zabbix-agent2) + mkdir -p /etc/zabbix + echo '{PSK_VALUE}' > /etc/zabbix/zabbix_agent2.psk + chmod 600 /etc/zabbix/zabbix_agent2.psk + chown -R zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true + echo '{conf_b64}' | base64 -d > /etc/zabbix/zabbix_agent2.conf + systemctl daemon-reload + systemctl enable --now zabbix-agent2.service + systemctl restart zabbix-agent2.service + systemctl is-active zabbix-agent2.service + """ + + stdin, stdout, stderr = ssh.exec_command(f"{sudo}bash -c \"{compound_cmd}\"") + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + + lines = out.splitlines() + last_line = lines[-1] if lines else "EMPTY" + print(f" [RESULT] {name} ({ip}) output: {last_line}") + if err: + print(" STDERR (tail):", err[-200:]) + + ssh.close() + except Exception as e: + print(f" [ERROR] Failed on {name} ({ip}): {e}") diff --git a/scripts/setup_awx_resources.py b/scripts/setup_awx_resources.py new file mode 100644 index 0000000..d1141f7 --- /dev/null +++ b/scripts/setup_awx_resources.py @@ -0,0 +1,183 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +orm_provisioning_code = """ +from awx.main.models import ( + Organization, Inventory, Group, Host, Credential, Project, JobTemplate, Schedule +) + +# 1. Organization +org, created = Organization.objects.get_or_create(name='Snarfnet Homelab', defaults={'description': 'Snarfnet Homelab Infrastructure Automation'}) +print(f"Organization: {org.name} (id={org.id}, created={created})") + +# 2. Inventory +inv, created = Inventory.objects.get_or_create( + name='Snarfnet Master Inventory', + defaults={'organization': org, 'description': 'Master Inventory across all 28 Homelab hosts'} +) +if not inv.organization: + inv.organization = org + inv.save() +print(f"Inventory: {inv.name} (id={inv.id}, created={created})") + +# 3. Groups & Hosts definition +GROUPS = { + 'k3s_aerodrome_masters': { + 'vars': {'ansible_user': 'root', 'cluster': 'aerodrome', 'node_role': 'master'}, + 'hosts': { + 'sninfr2lnxctl01': '140.44.4.11', + 'sninfr2lnxctl02': '140.44.4.12', + 'sninfr2lnxctl03': '140.44.4.13', + } + }, + 'k3s_aerodrome_workers': { + 'vars': {'ansible_user': 'root', 'cluster': 'aerodrome', 'node_role': 'worker'}, + 'hosts': { + 'sninfr2lnxwrk01': '140.44.4.21', + 'sninfr2lnxwrk02': '140.44.4.22', + 'sninfr2lnxwrk03': '140.44.4.23', + 'sninfr2lnxwrk04': '140.44.4.24', + 'sninfr2lnxwrk05': '140.44.4.25', + 'sninfr2lnxwrk06': '140.44.4.26', + 'sninfr2lnxwrk07': '140.44.4.27', + } + }, + 'k3s_spaceport_masters': { + 'vars': {'ansible_user': 'root', 'cluster': 'spaceport', 'node_role': 'master'}, + 'hosts': { + 'sninfr0lnxctl01': '140.44.4.41', + 'sninfr0lnxctl02': '140.44.4.42', + 'sninfr0lnxctl03': '140.44.4.43', + 'sninfr0lnxctl04': '140.44.4.44', + } + }, + 'k3s_spaceport_workers': { + 'vars': {'ansible_user': 'root', 'cluster': 'spaceport', 'node_role': 'worker'}, + 'hosts': { + 'sninfr0lnxwrk01': '140.44.4.51', + 'sninfr0lnxwrk02': '140.44.4.52', + 'sninfr0lnxwrk03': '140.44.4.53', + 'sninfr0lnxwrk04': '140.44.4.54', + 'sninfr0lnxwrk05': '140.44.4.55', + 'sninfr0lnxwrk06': '140.44.4.56', + } + }, + 'swarm_shipyard': { + 'vars': {'ansible_user': 'snarf', 'ansible_become': True, 'cluster': 'shipyard'}, + 'hosts': { + 'snshpd4lnxswm01': '140.44.4.71', + 'snshpd4lnxswm02': '140.44.4.72', + 'snshpd4lnxswm03': '140.44.4.73', + 'snshpd4lnxswm04': '140.44.4.74', + 'snshpd4lnxswm05': '140.44.4.75', + } + }, + 'ai_pi_cluster': { + 'vars': {'ansible_user': 'snarf', 'ansible_become': True}, + 'hosts': { + 'sninfr4lnxaih01': '140.44.4.7', + 'sninfr4lnxaih02': '140.44.4.8', + } + }, + 'smart_home_nodes': { + 'vars': {'ansible_user': 'snarf', 'ansible_become': True}, + 'hosts': { + 'homeassistant': '14.4.4.20', + 'nutserver': '14.4.4.44', + } + }, + 'synology_nas': { + 'vars': {}, + 'hosts': { + 'sn_nas': '14.10.10.71', + 'snarfnas': '14.10.10.72', + } + } +} + +import yaml +for grp_name, data in GROUPS.items(): + grp, _ = Group.objects.get_or_create(name=grp_name, inventory=inv) + grp.variables = yaml.dump(data.get('vars', {})) + grp.save() + + for h_name, ip in data.get('hosts', {}).items(): + h, _ = Host.objects.get_or_create(name=h_name, inventory=inv) + h.variables = yaml.dump({'ansible_host': ip}) + h.save() + if not grp.hosts.filter(id=h.id).exists(): + grp.hosts.add(h) + +print(f"Total Hosts in '{inv.name}': {inv.hosts.count()}") +print(f"Total Groups in '{inv.name}': {inv.groups.count()}") + +# 4. Project +proj, created = Project.objects.get_or_create( + name='Snarfnet Infrastructure Playbooks', + defaults={ + 'organization': org, + 'scm_type': 'git', + 'scm_url': 'https://git.snarfnet.net/Snarf/SnarfCode.git', + 'scm_branch': 'main', + 'scm_clean': True, + 'scm_delete_on_update': True, + 'scm_update_on_launch': True, + 'description': 'Snarfnet Ansible Playbooks from local Gitea' + } +) +print(f"Project: {proj.name} (id={proj.id}, created={created})") + +# 5. Credentials lookup +cred_root = Credential.objects.filter(name__icontains='root').first() +cred_snarf = Credential.objects.filter(name__icontains='snarf').first() +print(f"Root Credential: {cred_root.name if cred_root else 'None'}") +print(f"Snarf Credential: {cred_snarf.name if cred_snarf else 'None'}") + +# 6. Job Templates +TEMPLATES = [ + { + 'name': 'Daily Drift & Agent Health Audit', + 'playbook': 'ansible/playbooks/daily_drift_audit.yml', + 'desc': 'Verifies time synchronization, Zabbix agent, and SSH access daily' + }, + { + 'name': 'Weekly OS & Security Patching', + 'playbook': 'ansible/playbooks/weekly_os_patching.yml', + 'desc': 'Rolling serial-1 apt update and dist-upgrade with reboot verification' + }, + { + 'name': 'Aerodrome K3s Pre-Upgrade Check', + 'playbook': 'ansible/playbooks/k3s_aerodrome_precheck.yml', + 'desc': 'Validates Aerodrome masters and creates on-demand etcd snapshot' + } +] + +for t in TEMPLATES: + jt, created = JobTemplate.objects.get_or_create( + name=t['name'], + defaults={ + 'project': proj, + 'inventory': inv, + 'playbook': t['playbook'], + 'description': t['desc'], + 'job_type': 'run', + } + ) + if cred_root and not jt.credentials.filter(id=cred_root.id).exists(): + jt.credentials.add(cred_root) + if cred_snarf and not jt.credentials.filter(id=cred_snarf.id).exists(): + jt.credentials.add(cred_snarf) + print(f"Job Template: {jt.name} (id={jt.id}, created={created})") + +print("AWX Provisioning Complete!") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{orm_provisioning_code}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +print(stderr.read().decode()) + +ssh.close()