feat(infra): Phase 3 & 4 completion, deploy AWX playbooks & Zabbix monitoring

This commit is contained in:
2026-11-02 04:17:38 -05:00
parent 9c3de8bb4d
commit 003ae80c9c
15 changed files with 711 additions and 2 deletions

49
scripts/audit_awx.py Normal file
View File

@@ -0,0 +1,49 @@
import json, ssl, urllib.request
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
AWX_URL = "http://140.44.4.30:87/api/v2"
ADMIN_USER = "admin"
ADMIN_PASS = "CKFj2RfRzNcIeNJIimoW2uZ9En25P4VL"
import base64
auth_header = "Basic " + base64.b64encode(f"{ADMIN_USER}:{ADMIN_PASS}".encode()).decode()
def awx_get(endpoint):
req = urllib.request.Request(f"{AWX_URL}/{endpoint}/", headers={"Authorization": auth_header})
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
return json.loads(r.read().decode())
try:
me = awx_get("me")
print("AWX Auth Success! User:", me["results"][0]["username"])
orgs = awx_get("organizations")
print(f"\nOrganizations ({orgs['count']}):")
for o in orgs["results"]:
print(f" - [{o['id']}] {o['name']}")
creds = awx_get("credentials")
print(f"\nCredentials ({creds['count']}):")
for c in creds["results"]:
print(f" - [{c['id']}] {c['name']} (Type: {c['credential_type']})")
inventories = awx_get("inventories")
print(f"\nInventories ({inventories['count']}):")
for inv in inventories["results"]:
print(f" - [{inv['id']}] {inv['name']}")
projects = awx_get("projects")
print(f"\nProjects ({projects['count']}):")
for p in projects["results"]:
print(f" - [{p['id']}] {p['name']} (Status: {p['status']}, SCM: {p['scm_type']})")
templates = awx_get("job_templates")
print(f"\nJob Templates ({templates['count']}):")
for t in templates["results"]:
print(f" - [{t['id']}] {t['name']}")
except Exception as e:
print("AWX Query Error:", e)

View File

@@ -12,3 +12,4 @@ stdin, stdout, stderr = ssh.exec_command('egrep "^(Server|ServerActive|Hostname|
print("CONFIG:\n", stdout.read().decode('utf-8', errors='replace'))
ssh.close()

View File

@@ -0,0 +1,13 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('kubectl get secrets -n awx')
print("=== AWX SECRETS ===\n", stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command("kubectl get secret awx-admin-password -n awx -o jsonpath='{.data.password}' | base64 -d")
print("ADMIN PASSWORD:\n", stdout.read().decode())
ssh.close()

View File

@@ -0,0 +1,12 @@
import paramiko, sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('grep -E "^[A-Za-z]" /etc/zabbix/zabbix_agent2.conf')
print("51 CONFIG:\n", stdout.read().decode('utf-8', errors='replace'))
ssh.close()

View File

@@ -0,0 +1,19 @@
import json, ssl, urllib.request
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
req = urllib.request.Request('http://140.44.4.60:8888/api_jsonrpc.php', data=json.dumps(payload).encode('utf-8'), headers={'Content-Type': 'application/json-rpc'}, method='POST')
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
token = json.loads(r.read().decode())['result']
payload = {'jsonrpc': '2.0', 'method': 'problem.get', 'params': {'output': ['eventid', 'name', 'severity', 'clock']}, 'id': 2}
req = urllib.request.Request('http://140.44.4.60:8888/api_jsonrpc.php', data=json.dumps(payload).encode('utf-8'), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'}, method='POST')
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
problems = json.loads(r.read().decode())['result']
print(f"Total Active Problems in Zabbix: {len(problems)}")
for p in problems:
print(f" - [Severity {p['severity']}] {p['name']}")

View File

@@ -0,0 +1,79 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
PSK_VALUE = "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0"
AGENT_CONF = """# Standardized Antigravity Managed Zabbix Agent 2 Config
Server=zabbix.snarfnet.net
ServerActive=zabbix.snarfnet.net
HostnameItem=system.hostname
HostMetadata=Linux
TLSConnect=psk
TLSAccept=psk
TLSPSKIdentity=PSK_autoregister
TLSPSKFile=/etc/zabbix/zabbix_agent2.psk
LogFile=/var/log/zabbix/zabbix_agent2.log
LogFileSize=10
BufferSend=5
BufferSize=100
"""
TARGETS = [
("sninfr0lnxctl01", "140.44.4.41", "root"),
("sninfr0lnxctl02", "140.44.4.42", "root"),
("sninfr0lnxctl03", "140.44.4.43", "root"),
("sninfr0lnxctl04", "140.44.4.44", "root"),
("sninfr2lnxwrk05", "140.44.4.25", "root"),
("sninfr2lnxwrk07", "140.44.4.27", "root"),
("sninfr4lnxaih01", "140.44.4.7", "snarf"),
("sninfr4lnxaih02", "140.44.4.8", "snarf"),
]
for name, ip, user in TARGETS:
print(f"\n=======================================================")
print(f"Deploying Zabbix Agent 2 to {name} ({ip}) as {user}...")
print(f"=======================================================")
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username=user, key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
sudo = "sudo " if user != "root" else ""
# 1. Check if zabbix-agent2 is installed, install if missing
stdin, stdout, stderr = ssh.exec_command(f"{sudo}which zabbix_agent2")
out = stdout.read().decode().strip()
if not out:
print(" zabbix-agent2 not found. Installing via apt...")
stdin, stdout, stderr = ssh.exec_command(f"{sudo}apt-get update -y && {sudo}apt-get install -y zabbix-agent2")
print(" Apt output:", stdout.read().decode()[-200:])
# 2. Write PSK file
stdin, stdout, stderr = ssh.exec_command(f"{sudo}mkdir -p /etc/zabbix")
stdin, stdout, stderr = ssh.exec_command(f"echo '{PSK_VALUE}' | {sudo}tee /etc/zabbix/zabbix_agent2.psk > /dev/null")
stdin, stdout, stderr = ssh.exec_command(f"{sudo}chmod 600 /etc/zabbix/zabbix_agent2.psk")
stdin, stdout, stderr = ssh.exec_command(f"{sudo}chown -R zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true")
# 3. Write conf file
# Using base64 to avoid escaping issues
import base64
conf_b64 = base64.b64encode(AGENT_CONF.encode('utf-8')).decode('ascii')
stdin, stdout, stderr = ssh.exec_command(f"echo '{conf_b64}' | base64 -d | {sudo}tee /etc/zabbix/zabbix_agent2.conf > /dev/null")
# 4. Restart and enable service
stdin, stdout, stderr = ssh.exec_command(f"{sudo}systemctl daemon-reload && {sudo}systemctl enable --now zabbix-agent2.service && {sudo}systemctl restart zabbix-agent2.service")
stdout.read()
# 5. Verify status
time.sleep(2)
stdin, stdout, stderr = ssh.exec_command(f"{sudo}systemctl is-active zabbix-agent2.service")
status = stdout.read().decode().strip()
print(f" [SUCCESS] {name} ({ip}) zabbix-agent2 service is: {status}")
ssh.close()
except Exception as e:
print(f" [ERROR] Failed on {name} ({ip}): {e}")

View File

@@ -0,0 +1,38 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """
from awx.main.models import Organization, Inventory, Group, Host, Credential, Project, JobTemplate
print("=== ORGANIZATIONS ===")
for o in Organization.objects.all():
print(f" [{o.id}] {o.name}")
print("\\n=== INVENTORIES ===")
for i in Inventory.objects.all():
print(f" [{i.id}] {i.name} (Hosts: {i.hosts.count()})")
print("\\n=== CREDENTIALS ===")
for c in Credential.objects.all():
print(f" [{c.id}] {c.name} (Type: {c.credential_type.name})")
print("\\n=== PROJECTS ===")
for p in Project.objects.all():
print(f" [{p.id}] {p.name} (SCM: {p.scm_type}, URL: {p.scm_url}, Status: {p.status})")
print("\\n=== JOB TEMPLATES ===")
for t in JobTemplate.objects.all():
print(f" [{t.id}] {t.name}")
"""
import base64
b64 = base64.b64encode(py_script.encode()).decode()
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_script}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
print(stderr.read().decode())
ssh.close()

View File

@@ -0,0 +1,61 @@
import json, ssl, urllib.request
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
def zbx(method, params, auth=None):
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
headers = {"Content-Type": "application/json-rpc"}
if auth:
headers["Authorization"] = f"Bearer {auth}"
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
resp = json.loads(r.read().decode())
if "error" in resp:
raise RuntimeError(f"Error in {method}: {resp['error']}")
return resp["result"]
token = zbx("user.login", {"username": "Admin", "password": "zabbix"})
print("Logged in!")
# 1. Create or get hostgroup 'Storage / NAS'
groups = zbx("hostgroup.get", {"filter": {"name": "Storage / NAS"}}, token)
if not groups:
res = zbx("hostgroup.create", {"name": "Storage / NAS"}, token)
groupid = res["groupids"][0]
print(f"Created hostgroup 'Storage / NAS': {groupid}")
else:
groupid = groups[0]["groupid"]
print(f"Found hostgroup 'Storage / NAS': {groupid}")
# 2. Register SN-NAS and SnarfNAS
nas_devices = [
{"host": "SN-NAS", "name": "SN-NAS (DS920+)", "ip": "14.10.10.71"},
{"host": "SnarfNAS", "name": "SnarfNAS (DS1821+)", "ip": "14.10.10.72"}
]
for nas in nas_devices:
existing = zbx("host.get", {"filter": {"host": nas["host"]}}, token)
if not existing:
res = zbx("host.create", {
"host": nas["host"],
"name": nas["name"],
"groups": [{"groupid": groupid}],
"templates": [{"templateid": "10564"}], # ICMP Ping
"interfaces": [{
"type": 1, # Agent / ICMP
"main": 1,
"useip": 1,
"ip": nas["ip"],
"dns": "",
"port": "10050"
}]
}, token)
print(f"Registered {nas['name']}: hostid={res['hostids'][0]}")
else:
print(f"{nas['name']} is already registered.")
print("All NAS devices registered in Zabbix!")

View File

@@ -0,0 +1,70 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
PSK_VALUE = "a1142332ccfa9c53040e36171bb551ceb2865fb4051d203404df39e88ebe94f0"
AGENT_CONF = """# Standardized Antigravity Managed Zabbix Agent 2 Config
Server=zabbix.snarfnet.net
ServerActive=zabbix.snarfnet.net
HostnameItem=system.hostname
HostMetadata=Linux
TLSConnect=psk
TLSAccept=psk
TLSPSKIdentity=PSK_autoregister
TLSPSKFile=/etc/zabbix/zabbix_agent2.psk
LogFile=/var/log/zabbix/zabbix_agent2.log
LogFileSize=10
BufferSend=5
BufferSize=100
"""
import base64
conf_b64 = base64.b64encode(AGENT_CONF.encode('utf-8')).decode('ascii')
TARGETS = [
("sninfr0lnxctl01", "140.44.4.41", "root"),
("sninfr0lnxctl02", "140.44.4.42", "root"),
("sninfr2lnxwrk05", "140.44.4.25", "root"),
("sninfr2lnxwrk07", "140.44.4.27", "root"),
]
for name, ip, user in TARGETS:
print(f"\n=======================================================")
print(f"Deploying Zabbix Agent 2 to {name} ({ip}) as {user}...")
print(f"=======================================================")
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username=user, key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
sudo = "sudo " if user != "root" else ""
compound_cmd = f"""
which zabbix_agent2 || (apt-get update -y && apt-get install -y zabbix-agent2)
mkdir -p /etc/zabbix
echo '{PSK_VALUE}' > /etc/zabbix/zabbix_agent2.psk
chmod 600 /etc/zabbix/zabbix_agent2.psk
chown -R zabbix:zabbix /etc/zabbix/zabbix_agent2.psk 2>/dev/null || true
echo '{conf_b64}' | base64 -d > /etc/zabbix/zabbix_agent2.conf
systemctl daemon-reload
systemctl enable --now zabbix-agent2.service
systemctl restart zabbix-agent2.service
systemctl is-active zabbix-agent2.service
"""
stdin, stdout, stderr = ssh.exec_command(f"{sudo}bash -c \"{compound_cmd}\"")
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
lines = out.splitlines()
last_line = lines[-1] if lines else "EMPTY"
print(f" [RESULT] {name} ({ip}) output: {last_line}")
if err:
print(" STDERR (tail):", err[-200:])
ssh.close()
except Exception as e:
print(f" [ERROR] Failed on {name} ({ip}): {e}")

View File

@@ -0,0 +1,183 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
orm_provisioning_code = """
from awx.main.models import (
Organization, Inventory, Group, Host, Credential, Project, JobTemplate, Schedule
)
# 1. Organization
org, created = Organization.objects.get_or_create(name='Snarfnet Homelab', defaults={'description': 'Snarfnet Homelab Infrastructure Automation'})
print(f"Organization: {org.name} (id={org.id}, created={created})")
# 2. Inventory
inv, created = Inventory.objects.get_or_create(
name='Snarfnet Master Inventory',
defaults={'organization': org, 'description': 'Master Inventory across all 28 Homelab hosts'}
)
if not inv.organization:
inv.organization = org
inv.save()
print(f"Inventory: {inv.name} (id={inv.id}, created={created})")
# 3. Groups & Hosts definition
GROUPS = {
'k3s_aerodrome_masters': {
'vars': {'ansible_user': 'root', 'cluster': 'aerodrome', 'node_role': 'master'},
'hosts': {
'sninfr2lnxctl01': '140.44.4.11',
'sninfr2lnxctl02': '140.44.4.12',
'sninfr2lnxctl03': '140.44.4.13',
}
},
'k3s_aerodrome_workers': {
'vars': {'ansible_user': 'root', 'cluster': 'aerodrome', 'node_role': 'worker'},
'hosts': {
'sninfr2lnxwrk01': '140.44.4.21',
'sninfr2lnxwrk02': '140.44.4.22',
'sninfr2lnxwrk03': '140.44.4.23',
'sninfr2lnxwrk04': '140.44.4.24',
'sninfr2lnxwrk05': '140.44.4.25',
'sninfr2lnxwrk06': '140.44.4.26',
'sninfr2lnxwrk07': '140.44.4.27',
}
},
'k3s_spaceport_masters': {
'vars': {'ansible_user': 'root', 'cluster': 'spaceport', 'node_role': 'master'},
'hosts': {
'sninfr0lnxctl01': '140.44.4.41',
'sninfr0lnxctl02': '140.44.4.42',
'sninfr0lnxctl03': '140.44.4.43',
'sninfr0lnxctl04': '140.44.4.44',
}
},
'k3s_spaceport_workers': {
'vars': {'ansible_user': 'root', 'cluster': 'spaceport', 'node_role': 'worker'},
'hosts': {
'sninfr0lnxwrk01': '140.44.4.51',
'sninfr0lnxwrk02': '140.44.4.52',
'sninfr0lnxwrk03': '140.44.4.53',
'sninfr0lnxwrk04': '140.44.4.54',
'sninfr0lnxwrk05': '140.44.4.55',
'sninfr0lnxwrk06': '140.44.4.56',
}
},
'swarm_shipyard': {
'vars': {'ansible_user': 'snarf', 'ansible_become': True, 'cluster': 'shipyard'},
'hosts': {
'snshpd4lnxswm01': '140.44.4.71',
'snshpd4lnxswm02': '140.44.4.72',
'snshpd4lnxswm03': '140.44.4.73',
'snshpd4lnxswm04': '140.44.4.74',
'snshpd4lnxswm05': '140.44.4.75',
}
},
'ai_pi_cluster': {
'vars': {'ansible_user': 'snarf', 'ansible_become': True},
'hosts': {
'sninfr4lnxaih01': '140.44.4.7',
'sninfr4lnxaih02': '140.44.4.8',
}
},
'smart_home_nodes': {
'vars': {'ansible_user': 'snarf', 'ansible_become': True},
'hosts': {
'homeassistant': '14.4.4.20',
'nutserver': '14.4.4.44',
}
},
'synology_nas': {
'vars': {},
'hosts': {
'sn_nas': '14.10.10.71',
'snarfnas': '14.10.10.72',
}
}
}
import yaml
for grp_name, data in GROUPS.items():
grp, _ = Group.objects.get_or_create(name=grp_name, inventory=inv)
grp.variables = yaml.dump(data.get('vars', {}))
grp.save()
for h_name, ip in data.get('hosts', {}).items():
h, _ = Host.objects.get_or_create(name=h_name, inventory=inv)
h.variables = yaml.dump({'ansible_host': ip})
h.save()
if not grp.hosts.filter(id=h.id).exists():
grp.hosts.add(h)
print(f"Total Hosts in '{inv.name}': {inv.hosts.count()}")
print(f"Total Groups in '{inv.name}': {inv.groups.count()}")
# 4. Project
proj, created = Project.objects.get_or_create(
name='Snarfnet Infrastructure Playbooks',
defaults={
'organization': org,
'scm_type': 'git',
'scm_url': 'https://git.snarfnet.net/Snarf/SnarfCode.git',
'scm_branch': 'main',
'scm_clean': True,
'scm_delete_on_update': True,
'scm_update_on_launch': True,
'description': 'Snarfnet Ansible Playbooks from local Gitea'
}
)
print(f"Project: {proj.name} (id={proj.id}, created={created})")
# 5. Credentials lookup
cred_root = Credential.objects.filter(name__icontains='root').first()
cred_snarf = Credential.objects.filter(name__icontains='snarf').first()
print(f"Root Credential: {cred_root.name if cred_root else 'None'}")
print(f"Snarf Credential: {cred_snarf.name if cred_snarf else 'None'}")
# 6. Job Templates
TEMPLATES = [
{
'name': 'Daily Drift & Agent Health Audit',
'playbook': 'ansible/playbooks/daily_drift_audit.yml',
'desc': 'Verifies time synchronization, Zabbix agent, and SSH access daily'
},
{
'name': 'Weekly OS & Security Patching',
'playbook': 'ansible/playbooks/weekly_os_patching.yml',
'desc': 'Rolling serial-1 apt update and dist-upgrade with reboot verification'
},
{
'name': 'Aerodrome K3s Pre-Upgrade Check',
'playbook': 'ansible/playbooks/k3s_aerodrome_precheck.yml',
'desc': 'Validates Aerodrome masters and creates on-demand etcd snapshot'
}
]
for t in TEMPLATES:
jt, created = JobTemplate.objects.get_or_create(
name=t['name'],
defaults={
'project': proj,
'inventory': inv,
'playbook': t['playbook'],
'description': t['desc'],
'job_type': 'run',
}
)
if cred_root and not jt.credentials.filter(id=cred_root.id).exists():
jt.credentials.add(cred_root)
if cred_snarf and not jt.credentials.filter(id=cred_snarf.id).exists():
jt.credentials.add(cred_snarf)
print(f"Job Template: {jt.name} (id={jt.id}, created={created})")
print("AWX Provisioning Complete!")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{orm_provisioning_code}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
print(stderr.read().decode())
ssh.close()