feat: add infrastructure automation and diagnostic scripts for Zabbix, Nginx, and Authentik

docs(topology): regenerate high-res PNG diagram to reflect latest layout, hardware and storage metrics
chore: update homelab-gitops-repo submodule reference
2026-11-12 20:18:37 -05:00 · 2026-11-12 17:13:51 -05:00 · 2026-11-12 17:09:51 -05:00 · 2026-11-12 17:08:14 -05:00 · 2026-11-12 16:57:08 -05:00 · 2026-11-08 15:03:27 -05:00
296 changed files with 10402 additions and 54 deletions
--- a/docker-deployments/heimdall.yml
+++ b/docker-deployments/heimdall.yml
@@ -1,20 +1,33 @@
+version: "3.8"
+
 services:
  heimdall:
    image: lscr.io/linuxserver/heimdall:latest
    ports:
-      - "85:80"
-      - "403:443"
+      - target: 80
+        published: 85
+        mode: ingress
+      - target: 443
+        published: 403
+        mode: ingress
    environment:
      - PUID=1000
      - PGID=1000
      - TZ=America/New_York
    volumes:
-      - heimdall_config:/config
+      - type: bind
+        source: /var/lib/docker-data/heimdall/config
+        target: /config
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          - node.hostname == snshpd4lnxswm01
+      restart_policy:
+        condition: on-failure

-volumes:
-  heimdall_config:
-    driver: local
-    driver_opts:
-      type: nfs
-      o: "addr=14.10.10.71,rw,nfsvers=4"
-      device: ":/volume1/docker/heimdall/"
+networks:
+  default:
+    name: homelab_public
+    external: true
--- a/docker-deployments/shepherd.yml
+++ b/docker-deployments/shepherd.yml
@@ -0,0 +1,21 @@
+version: "3.8"
+
+services:
+  shepherd:
+    image: containrrr/shepherd:latest
+    environment:
+      - SLEEP_TIME=24h
+      - IGNORELIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup
+      - UPDATE_OPTIONS=--with-registry-auth
+      - ROLLBACK_FAILURE=true
+      - TZ=America/New_York
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          - node.role == manager
+      restart_policy:
+        condition: on-failure
--- a/docs/snarfnet_homelab_topology.html
+++ b/docs/snarfnet_homelab_topology.html
--- a/docs/snarfnet_homelab_topology.png
+++ b/docs/snarfnet_homelab_topology.png
--- a/2
+++ b/2
--- a/scripts/add_forward_rules.py
+++ b/scripts/add_forward_rules.py
@@ -0,0 +1,22 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f">>> {cmd}")
+    _, o, e = ssh.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    if out:
+        print(out)
+    if err:
+        print("[stderr]", err)
+
+run("iptables -I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT")
+run("iptables -I FORWARD 2 -p tcp -d 140.44.4.64 --dport 10051 -j ACCEPT")
+run("iptables -L FORWARD -n -v | head -n 8")
+
+ssh.close()
+
--- a/scripts/add_kuma_bookmarks.py
+++ b/scripts/add_kuma_bookmarks.py
@@ -0,0 +1,58 @@
+import yaml
+
+def str_presenter(dumper, data):
+    if '\n' in data:
+        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+
+yaml.add_representer(str, str_presenter)
+yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+docs = list(yaml.safe_load_all(text))
+cm = docs[2]
+
+bookmarks_yaml = """- Quick Links:
+    - GitHub:
+        - abbr: GH
+          href: https://github.com/
+    - UniFi Site Manager:
+        - abbr: UI
+          href: https://unifi.ui.com/
+    - Cloudflare:
+        - abbr: CF
+          href: https://dash.cloudflare.com/
+- Grafana Telemetry:
+    - Systems Performance:
+        - abbr: SYS
+          href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861
+    - Container Workloads:
+        - abbr: K8S
+          href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads
+    - AI Stack Intelligence:
+        - abbr: AI
+          href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence
+- Uptime Status Pages:
+    - Unified Overview:
+        - abbr: ALL
+          href: https://uptime.snarfnet.net/status/default
+    - Core & VIP Fabric:
+        - abbr: VIP
+          href: https://uptime.snarfnet.net/status/core
+    - GitOps & CI/CD:
+        - abbr: OPS
+          href: https://uptime.snarfnet.net/status/gitops
+    - Edge & Ingress:
+        - abbr: EDG
+          href: https://uptime.snarfnet.net/status/edge
+"""
+
+cm['data']['bookmarks.yaml'] = bookmarks_yaml
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
+    yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+print("Updated bookmarks in homepage.yaml successfully!")
+
--- a/scripts/add_media_to_snarf.py
+++ b/scripts/add_media_to_snarf.py
@@ -0,0 +1,32 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
+
+payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
+token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+media_payload = {
+    'jsonrpc': '2.0',
+    'method': 'user.update',
+    'params': {
+        'userid': '3',
+        'medias': [
+            {
+                'mediatypeid': '70',
+                'sendto': 'snarf@snarfnet.net',
+                'active': 0,
+                'severity': 63, # 1+2+4+8+16+32
+                'period': '1-7,00:00-24:00'
+            }
+        ]
+    },
+    'id': 2
+}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(media_payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
+res = json.loads(urllib.request.urlopen(req, context=ctx).read())
+print("User update result:", res)
--- a/scripts/add_topology_to_homepage.py
+++ b/scripts/add_topology_to_homepage.py
@@ -0,0 +1,26 @@
+import yaml
+
+def str_presenter(dumper, data):
+    if '\n' in data:
+        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+
+yaml.add_representer(str, str_presenter)
+yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+docs = list(yaml.safe_load_all(text))
+cm = docs[2]
+
+services = yaml.safe_load(cm['data']['services.yaml'])
+bookmarks = yaml.safe_load(cm['data']['bookmarks.yaml'])
+
+cm['data']['services.yaml'] = yaml.dump(services, sort_keys=False, default_flow_style=False, allow_unicode=True)
+cm['data']['bookmarks.yaml'] = yaml.dump(bookmarks, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
+    yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+print("Cleaned up unicode strings in homepage.yaml!")
--- a/scripts/add_zabbix_devices.py
+++ b/scripts/add_zabbix_devices.py
@@ -0,0 +1,118 @@
+import json
+import ssl
+import urllib.request
+
+ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
+ZABBIX_USER = "Admin"
+ZABBIX_PASS = "zabbix"
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+def zbx_call(method, params, auth=None):
+    payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
+    headers = {"Content-Type": "application/json-rpc"}
+    if auth:
+        headers["Authorization"] = f"Bearer {auth}"
+    req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
+    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+        resp = json.loads(r.read().decode("utf-8"))
+        if "error" in resp:
+            raise RuntimeError(f"Zabbix error in {method}: {resp['error']}")
+        return resp["result"]
+
+auth = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS})
+
+# Check existing host names
+existing_hosts = zbx_call("host.get", {"output": ["hostid", "host", "name"]}, auth)
+existing_names = {h["host"].lower(): h["hostid"] for h in existing_hosts}
+
+# Create host groups if not exist
+groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth)
+group_map = {g["name"]: g["groupid"] for g in groups}
+
+for gname in ["Network Devices", "Environmental & Sensors", "Management & IPMI"]:
+    if gname not in group_map:
+        res = zbx_call("hostgroup.create", {"name": gname}, auth)
+        group_map[gname] = res["groupids"][0]
+        print(f"Created Host Group: {gname} (ID: {group_map[gname]})")
+
+devices_to_add = [
+    {
+        "host": "idrac-spacedock4",
+        "name": "Dell PowerEdge R730 Node 4 iDRAC (idrac-spacedock4)",
+        "ip": "140.44.44.124",
+        "group": "Management & IPMI"
+    },
+    {
+        "host": "usw-agg-core",
+        "name": "UniFi Switch Aggregation 10G Core",
+        "ip": "14.44.4.26",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-24-poe-closet",
+        "name": "UniFi Switch 24 PoE Closet",
+        "ip": "14.44.4.61",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-pro-24-poe-compute",
+        "name": "UniFi Switch Pro 24 PoE Compute",
+        "ip": "14.44.4.107",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-flex-attic",
+        "name": "UniFi Switch Flex Attic",
+        "ip": "14.44.4.98",
+        "group": "Network Devices"
+    },
+    {
+        "host": "usw-flex-garage",
+        "name": "UniFi Switch Flex Garage",
+        "ip": "14.44.4.96",
+        "group": "Network Devices"
+    },
+    {
+        "host": "tempest-weather-station",
+        "name": "Tempest Weather Station Gateway",
+        "ip": "14.4.4.47",
+        "group": "Environmental & Sensors"
+    },
+    {
+        "host": "sense-energy-monitor",
+        "name": "Sense Home Energy Gateway",
+        "ip": "14.4.4.48",
+        "group": "Environmental & Sensors"
+    }
+]
+
+for dev in devices_to_add:
+    hname = dev["host"]
+    if hname.lower() in existing_names:
+        print(f"[EXISTS] {hname}")
+        continue
+    params = {
+        "host": dev["host"],
+        "name": dev["name"],
+        "interfaces": [
+            {
+                "type": 1, # Agent / ICMP
+                "main": 1,
+                "useip": 1,
+                "ip": dev["ip"],
+                "dns": "",
+                "port": "10050"
+            }
+        ],
+        "groups": [{"groupid": group_map[dev["group"]]}],
+        "templates": [{"templateid": "10564"}] # ICMP Ping
+    }
+    try:
+        res = zbx_call("host.create", params, auth)
+        print(f"[CREATED] {dev['name']} ({dev['ip']}) -> HostID {res['hostids'][0]}")
+    except Exception as e:
+        print(f"[FAILED] {dev['name']}: {e}")
+
--- a/scripts/apply_full_homepage.py
+++ b/scripts/apply_full_homepage.py
@@ -0,0 +1,250 @@
+import yaml
+
+def str_presenter(dumper, data):
+    if '\n' in data:
+        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+
+yaml.add_representer(str, str_presenter)
+yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+docs = list(yaml.safe_load_all(text))
+cm = docs[2]
+
+services_yaml = """- Critical Services:
+    - Authentik:
+        icon: authentik.png
+        href: https://authentik.snarfnet.net
+        description: SSO & Identity Management
+    - PeaNUT UPS:
+        icon: peanut.png
+        href: https://portal.snarfnet.net
+        description: UPS Power & Battery Monitor
+    - Unifi Controller:
+        icon: unifi.png
+        href: https://unifi.ui.com
+        widget:
+          type: unifi
+          url: https://140.44.44.1
+          username: Snarf
+          password: Katzir476!!!
+
+- GitOps & Automation:
+    - Gitea:
+        icon: gitea.png
+        href: https://git.snarfnet.net
+        description: Git Repositories
+        widget:
+          type: gitea
+          url: https://git.snarfnet.net
+          key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71
+    - ArgoCD:
+        icon: argocd.png
+        href: https://argo.snarfnet.net
+        description: Declarative GitOps Reconciler
+        widget:
+          type: argocd
+          url: https://argo.snarfnet.net
+          key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0
+    - AWX Automation:
+        icon: awx.png
+        href: https://awx.snarfnet.net
+        description: Ansible Automation Controller
+    - Semaphore:
+        icon: semaphore.png
+        href: https://semaphore.snarfnet.net
+        description: Playbook Execution Engine
+
+- Storage & Clusters:
+    - Longhorn Aerodrome:
+        icon: longhorn.png
+        href: https://longhorn.aerodrome.snarfnet.net
+        description: Aerodrome Distributed Flash Storage
+    - Longhorn Spaceport:
+        icon: longhorn.png
+        href: https://longhorn.spaceport.snarfnet.net
+        description: Spaceport Distributed Block Storage
+    - Rancher (Aerodrome):
+        icon: rancher.png
+        href: https://aerodrome.snarfnet.net
+        description: Kubernetes Management Console
+        widget:
+          type: customapi
+          url: https://aerodrome.snarfnet.net/v3/clusters/local
+          headers:
+            Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
+          mappings:
+            - field: capacity.cpu
+              label: CPU Cores
+            - field: capacity.pods
+              label: Max Pods
+            - field: state
+              label: Cluster State
+    - Harvester HCI (Spacedock):
+        icon: harvester.png
+        href: https://harvester.spacedock.snarfnet.net
+        description: HCI Virtualization Hypervisors
+        widget:
+          type: customapi
+          url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt
+          headers:
+            Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
+          mappings:
+            - field: capacity.cpu
+              label: CPU Cores
+            - field: capacity.pods
+              label: Max Pods
+            - field: state
+              label: HCI State
+    - Portainer Swarm:
+        icon: portainer.png
+        href: https://shipyard.snarfnet.net
+        description: Docker Swarm Container Fleet
+        widget:
+          type: portainer
+          url: https://140.44.4.71:9443
+          env: 3
+          key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8=
+
+- Infrastructure & Observability:
+    - NGINX Proxy Manager:
+        icon: nginx-proxy-manager
+        href: https://nginx.snarfnet.net
+        description: Reverse Proxy & SSL Certificates
+    - Uptime Kuma:
+        icon: uptime-kuma
+        href: https://uptime.snarfnet.net
+        description: Multi-Endpoint Availability Probes
+        widget:
+          type: uptimekuma
+          url: http://140.44.4.28:448
+          slug: default
+    - Zabbix:
+        icon: zabbix.png
+        href: https://zabbix.snarfnet.net
+        description: Infrastructure Metrics & Alarms
+    - Grafana:
+        icon: grafana.png
+        href: https://grafana.snarfnet.net
+        description: Real-Time Metrics & AI Dashboards
+        widget:
+          type: grafana
+          url: https://grafana.snarfnet.net
+          username: admin
+          password: Katzir476!!!
+    - Core Node Glances:
+        icon: glances.png
+        href: http://140.44.4.11:61208
+        description: Host Resource Monitor
+
+- Environment & Weather:
+    - Tempest Weather Station:
+        icon: weather.png
+        href: https://snarfhome.snarfnet.net
+        description: Backyard Weather Telemetry
+        widget:
+          type: homeassistant
+          url: http://14.4.4.20:8123
+          key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4
+          custom:
+            - template: "{{ states('sensor.st_00158555_temperature') | float(0) | round(1) }} °F"
+              label: Outdoor Temp
+            - template: "{{ states('sensor.st_00158555_feels_like') | float(0) | round(1) }} °F"
+              label: Feels Like
+            - template: "{{ states('sensor.st_00158555_humidity') | float(0) | round(1) }} %"
+              label: Humidity
+            - template: "{{ states('sensor.st_00158555_wind_speed') | float(0) | round(1) }} mph"
+              label: Wind Speed
+"""
+
+settings_yaml = """title: SnarfNet Operations & Telemetry
+headerStyle: boxed
+color: slate
+theme: dark
+statusStyle: badge
+background:
+  image: https://i.imgur.com/hOzzkCY.jpeg
+  blur: sm
+  opacity: 50
+  brightness: 50
+cardBlur: sm
+quicklaunch:
+  searchDescriptions: true
+  hideInternetSearch: true
+  hideVisitURL: true
+  hideVersion: true
+layout:
+  Critical Services:
+    icon: mdi-shield-check
+    style: row
+    columns: 3
+  GitOps & Automation:
+    icon: mdi-git
+    style: row
+    columns: 4
+  Storage & Clusters:
+    icon: mdi-server-network
+    style: row
+    columns: 3
+  Infrastructure & Observability:
+    icon: mdi-chart-timeline-variant
+    style: row
+    columns: 3
+  Environment & Weather:
+    icon: mdi-weather-partly-cloudy
+    style: row
+    columns: 1
+showStats: true
+providers:
+  longhorn:
+    url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500
+"""
+
+widgets_yaml = """- kubernetes:
+    cluster:
+      show: true
+      cpu: true
+      memory: true
+      show_names: true
+    nodes:
+      show: false
+- longhorn:
+    total: true
+    nodes: false
+- resources:
+    label: System
+    backend: resources
+    cpu: true
+    expanded: true
+    memory: true
+    disk: /
+- search:
+    provider: duckduckgo
+    target: _blank
+"""
+
+bookmarks_yaml = """- Quick Links:
+    - GitHub:
+        - abbr: GH
+          href: https://github.com/
+    - UniFi Site Manager:
+        - abbr: UI
+          href: https://unifi.ui.com/
+    - Cloudflare:
+        - abbr: CF
+          href: https://dash.cloudflare.com/
+"""
+
+cm['data']['services.yaml'] = services_yaml
+cm['data']['settings.yaml'] = settings_yaml
+cm['data']['widgets.yaml'] = widgets_yaml
+cm['data']['bookmarks.yaml'] = bookmarks_yaml
+cm['data']['kubernetes.yaml'] = "mode: cluster\n"
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
+    yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
+
+print("Saved clean homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml successfully!")
--- a/scripts/apply_grafana_dashboards.py
+++ b/scripts/apply_grafana_dashboards.py
@@ -0,0 +1,33 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r", encoding="utf-8") as f:
+    grafana_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_updated.yaml")
+stdin.write(grafana_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying Grafana manifests:")
+print(run("kubectl apply -f /tmp/grafana_updated.yaml"))
+
+print("\nRestarting Grafana:")
+print(run("kubectl rollout restart deployment/grafana -n default"))
+print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
+
+print("\nWaiting for Grafana to finish initialization...")
+time.sleep(10)
+
+c.close()
+
--- a/scripts/apply_grafana_server_side.py
+++ b/scripts/apply_grafana_server_side.py
@@ -0,0 +1,20 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+print("Applying server-side:")
+print(run("kubectl apply -f /tmp/grafana_full.yaml --server-side --force-conflicts"))
+print("\nRollout restart:")
+print(run("kubectl rollout restart deployment/grafana -n default"))
+print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
+
+c.close()
+
--- a/scripts/apply_h21_authentik.py
+++ b/scripts/apply_h21_authentik.py
@@ -0,0 +1,149 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect("140.44.4.71", username="root", key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+adv_config = """# Buffer tuning for Authentik headers
+proxy_buffers 8 16k;
+proxy_buffer_size 32k;
+
+# Outpost verification route - plain HTTP to internal VIP
+location /outpost.goauthentik.io {
+    auth_request off;
+    proxy_pass              http://140.44.4.29:9000/outpost.goauthentik.io;
+    proxy_set_header        Host authentik.snarfnet.net;
+    proxy_set_header        X-Original-URL $scheme://$http_host$request_uri;
+    add_header              Set-Cookie $auth_cookie;
+    auth_request_set        $auth_cookie $upstream_http_set_cookie;
+    proxy_pass_request_body off;
+    proxy_set_header        Content-Length "";
+}
+
+# Redirect to Authentik when unauthenticated
+location @goauthentik_proxy_signin {
+    internal;
+    auth_request off;
+    add_header Set-Cookie $auth_cookie;
+    return 302 https://authentik.snarfnet.net/outpost.goauthentik.io/start?rd=$scheme://$http_host$request_uri;
+}
+
+# Protected Topology HTML (Interactive)
+location = /topology.html {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.html;
+    default_type text/html;
+}
+
+# Protected Topology PNG (High-Res Image)
+location = /topology.png {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.png;
+    default_type image/png;
+}
+
+location = /snarfnet_homelab_topology.html {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.html;
+    default_type text/html;
+}
+
+location = /snarfnet_homelab_topology.png {
+    auth_request /outpost.goauthentik.io/auth/nginx;
+    error_page 401 = @goauthentik_proxy_signin;
+    auth_request_set $auth_cookie $upstream_http_set_cookie;
+    add_header Set-Cookie $auth_cookie;
+    add_header Cache-Control "no-cache, no-store, must-revalidate";
+    expires 0;
+
+    alias /data/topology/snarfnet_homelab_topology.png;
+    default_type image/png;
+}
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/adv_config_h21.txt")
+stdin.write(adv_config)
+stdin.close()
+stdout.read()
+
+remote_script = """import sqlite3, shutil, os
+
+db_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'
+conf_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/21.conf'
+
+with open('/tmp/adv_config_h21.txt', 'r') as f:
+    adv = f.read()
+
+# Update DB
+conn = sqlite3.connect(db_path)
+cur = conn.cursor()
+cur.execute("UPDATE proxy_host SET advanced_config = ? WHERE id = 21", (adv,))
+conn.commit()
+conn.close()
+print("Updated database for host 21.")
+
+# Update 21.conf
+with open(conf_path, 'r') as f:
+    conf = f.read()
+
+# Strip existing adv_config block if present
+tag = '# Buffer tuning for Authentik headers'
+if tag in conf:
+    idx_start = conf.find(tag)
+    idx_end = conf.find('location / {', idx_start)
+    if idx_start != -1 and idx_end != -1:
+        new_conf = conf[:idx_start] + adv + "\\n\\n  " + conf[idx_end:]
+    else:
+        new_conf = conf
+else:
+    idx = conf.find('location / {')
+    if idx != -1:
+        new_conf = conf[:idx] + adv + "\\n\\n  " + conf[idx:]
+    else:
+        new_conf = conf.rstrip().rstrip('}') + "\\n\\n" + adv + "\\n}\\n"
+
+with open(conf_path, 'w') as f:
+    f.write(new_conf)
+print("Updated 21.conf.")
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/apply_h21.py")
+stdin.write(remote_script)
+stdin.close()
+stdout.read()
+
+stdin, stdout, stderr = c.exec_command("python3 /tmp/apply_h21.py")
+print("Remote python stdout:", stdout.read().decode())
+print("Remote python stderr:", stderr.read().decode())
+
+stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -t")
+out = stdout.read().decode()
+err = stderr.read().decode()
+print("nginx -t stdout:", out)
+print("nginx -t stderr:", err)
+
+if "successful" in out or "successful" in err:
+    print("Nginx syntax OK! Reloading nginx...")
+    stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
+    print("Reload stdout:", stdout.read().decode())
+    print("Reload stderr:", stderr.read().decode())
+
+c.close()
--- a/scripts/apply_homepage.py
+++ b/scripts/apply_homepage.py
@@ -0,0 +1,31 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    if out:
+        print(out)
+    if err:
+        print("[stderr]", err)
+    return out
+
+with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+stdin, stdout, stderr = c.exec_command('cat > /tmp/homepage.yaml')
+stdin.write(content)
+stdin.close()
+stdout.read()
+
+run("ls -lh /tmp/homepage.yaml")
+run("kubectl apply -f /tmp/homepage.yaml")
+run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure")
+run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s")
+
+c.close()
--- a/scripts/apply_prom_grafana.py
+++ b/scripts/apply_prom_grafana.py
@@ -0,0 +1,45 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r") as f:
+    prom_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml")
+stdin.write(prom_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying Prometheus:")
+print(run("kubectl apply -f /tmp/prometheus.yaml"))
+
+with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r") as f:
+    grafana_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_full.yaml")
+stdin.write(grafana_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("\nApplying Grafana configuration & dashboards:")
+print(run("kubectl apply -f /tmp/grafana_full.yaml"))
+
+print("\nRestarting Grafana:")
+print(run("kubectl rollout restart deployment/grafana -n default"))
+print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
+
+print("\nChecking Prometheus pod:")
+time.sleep(5)
+print(run("kubectl get pods -l app.kubernetes.io/name=prometheus"))
+
+c.close()
+
--- a/scripts/apply_prom_update.py
+++ b/scripts/apply_prom_update.py
@@ -0,0 +1,36 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f:
+    prom_yaml = f.read()
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_updated.yaml")
+stdin.write(prom_yaml)
+stdin.close()
+stdout.channel.recv_exit_status()
+
+print("Applying Prometheus:")
+print(run("kubectl apply -f /tmp/prometheus_updated.yaml"))
+
+print("\nRestarting Prometheus:")
+print(run("kubectl rollout restart deployment/prometheus -n default"))
+print(run("kubectl rollout status deployment/prometheus -n default --timeout=60s"))
+
+print("\nWaiting 15 seconds for scrape loop...")
+time.sleep(15)
+
+print("\nPrometheus active targets count:")
+print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets | grep -o '\"health\":\"up\"' | wc -l"))
+
+c.close()
+
--- a/scripts/audit_zabbix_alerts.py
+++ b/scripts/audit_zabbix_alerts.py
@@ -0,0 +1,48 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
+
+def zbx_req(method, params, token=None):
+    payload = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1}
+    headers = {'Content-Type': 'application/json-rpc'}
+    if token:
+        headers['Authorization'] = f'Bearer {token}'
+    req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method='POST')
+    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+        return json.loads(r.read().decode())['result']
+
+token = zbx_req('user.login', {'username': 'Admin', 'password': 'zabbix'})
+print("Zabbix Auth Token:", token)
+
+# 1. Actions
+actions = zbx_req('action.get', {'output': 'extend', 'selectOperations': 'extend', 'selectFilter': 'extend'}, token)
+print("\n=== TRIGGER ACTIONS ===")
+for a in actions:
+    print(f"ID: {a['actionid']}, Name: {a['name']}, Status (0=enabled, 1=disabled): {a['status']}, EventSource: {a['eventsource']}")
+    print(f"  Operations: {a.get('operations')}")
+    print(f"  Filter: {a.get('filter')}")
+
+# 2. Media Types
+media_types = zbx_req('mediatype.get', {'output': 'extend'}, token)
+print("\n=== MEDIA TYPES ===")
+for m in media_types:
+    print(f"ID: {m['mediatypeid']}, Name: {m['name']}, Type: {m['type']}, Status: {m['status']}")
+
+# 3. User Media
+users = zbx_req('user.get', {'output': ['userid', 'username'], 'selectMedias': 'extend'}, token)
+print("\n=== USERS & MEDIA ===")
+for u in users:
+    print(f"User: {u['username']} (ID: {u['userid']})")
+    for med in u.get('medias', []):
+        print(f"  Media: mediatypeid={med['mediatypeid']}, sendto={med['sendto']}, active={med['active']}, severity={med['severity']}")
+
+# 4. Recent Alerts
+alerts = zbx_req('alert.get', {'output': 'extend', 'limit': 15, 'sortfield': 'clock', 'sortorder': 'DESC'}, token)
+print("\n=== RECENT ALERTS ===")
+for al in alerts:
+    print(f"Alert ID: {al['alertid']}, ActionID: {al['actionid']}, Clock: {al['clock']}, Error: {al.get('error')}, Status: {al.get('status')}, Subject: {al.get('subject')}")
+
--- a/scripts/capture_screenshot.py
+++ b/scripts/capture_screenshot.py
@@ -0,0 +1,27 @@
+import subprocess
+import os
+
+edge_path = r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe"
+out_png = os.path.abspath("docs/snarfnet_homelab_topology.png")
+html_url = "file:///" + os.path.abspath("docs/snarfnet_homelab_topology.html").replace("\\", "/")
+
+cmd = [
+    edge_path,
+    "--headless",
+    "--disable-gpu",
+    "--no-sandbox",
+    "--hide-scrollbars",
+    "--window-size=1920,2400",
+    f"--screenshot={out_png}",
+    html_url
+]
+
+print("Running cmd:", " ".join(cmd))
+res = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+print("Return code:", res.returncode)
+print("Stdout:", res.stdout)
+print("Stderr:", res.stderr)
+print("File exists?", os.path.exists(out_png))
+if os.path.exists(out_png):
+    print("File size:", os.path.getsize(out_png))
+
--- a/scripts/cat_21_conf.py
+++ b/scripts/cat_21_conf.py
@@ -0,0 +1,10 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+cmd = "docker exec $(docker ps -q --filter name=nginx-proxy-manager_app) cat /data/nginx/proxy_host/21.conf"
+_, o, _ = c.exec_command(cmd)
+print(o.read().decode())
+c.close()
+
--- a/scripts/cat_prom_conf.py
+++ b/scripts/cat_prom_conf.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl exec deployment/prometheus -- cat /etc/prometheus/prometheus.yml"))
+c.close()
+
--- a/scripts/check_aero_metrics_sources.py
+++ b/scripts/check_aero_metrics_sources.py
@@ -0,0 +1,24 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Nodes on Aerodrome:")
+print(run("kubectl get nodes -o wide"))
+
+print("\nDaemonSets on Aerodrome:")
+print(run("kubectl get ds -A"))
+
+print("\nServices matching metrics:")
+print(run("kubectl get svc -A | grep -E 'metric|node|cadvisor|exporter'"))
+
+print("\nKubelet metrics on node 140.44.4.11 (port 10250 or 10255):")
+print(run("curl -sk https://localhost:10250/metrics/cadvisor | head -n 10"))
+
+c.close()
+
--- a/scripts/check_alerts_status.py
+++ b/scripts/check_alerts_status.py
@@ -0,0 +1,19 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
+
+payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
+token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+payload = {'jsonrpc': '2.0', 'method': 'alert.get', 'params': {'output': 'extend', 'limit': 20, 'sortfield': 'clock', 'sortorder': 'DESC'}, 'id': 2}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
+alerts = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+for a in alerts:
+    print(f"ID: {a['alertid']} | Status: {a['status']} | User: {a['userid']} | Error: '{a.get('error')}' | Subject: {a.get('subject')}")
+
--- a/scripts/check_argo_secret.py
+++ b/scripts/check_argo_secret.py
@@ -0,0 +1,16 @@
+import paramiko
+import base64
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Argo initial secret:")
+print(run("kubectl get secret argocd-initial-admin-secret -n infrastructure -o jsonpath='{.data.password}'"))
+
+c.close()
+
--- a/scripts/check_argocd_apps.py
+++ b/scripts/check_argocd_apps.py
@@ -0,0 +1,73 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+try:
+    resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
+    data = resp.json()
+    items = data.get("items", [])
+    print(f"Total Applications: {len(items)}\n")
+    
+    out_of_sync = []
+    synced = []
+    
+    for app in items:
+        name = app.get("metadata", {}).get("name")
+        sync_status = app.get("status", {}).get("sync", {}).get("status")
+        health_status = app.get("status", {}).get("health", {}).get("status")
+        dest_server = app.get("spec", {}).get("destination", {}).get("server")
+        dest_ns = app.get("spec", {}).get("destination", {}).get("namespace")
+        repo = app.get("spec", {}).get("source", {}).get("repoURL")
+        path = app.get("spec", {}).get("source", {}).get("path")
+        
+        # Check operation or sync conditions/messages
+        operation_state = app.get("status", {}).get("operationState", {})
+        op_phase = operation_state.get("phase")
+        op_msg = operation_state.get("message")
+        conditions = app.get("status", {}).get("conditions", [])
+        
+        info = {
+            "name": name,
+            "sync": sync_status,
+            "health": health_status,
+            "dest_ns": dest_ns,
+            "dest_server": dest_server,
+            "repo": repo,
+            "path": path,
+            "op_phase": op_phase,
+            "op_msg": op_msg,
+            "conditions": conditions
+        }
+        
+        if sync_status != "Synced" or health_status not in ["Healthy", None]:
+            out_of_sync.append(info)
+        else:
+            synced.append(info)
+            
+    print(f"=== OUT OF SYNC / DEGRADED APPS ({len(out_of_sync)}) ===")
+    for a in out_of_sync:
+        print(f"\nApp: {a['name']}")
+        print(f"  Sync: {a['sync']} | Health: {a['health']}")
+        print(f"  Dest: {a['dest_ns']} on {a['dest_server']}")
+        print(f"  Path: {a['path']}")
+        if a['op_phase']:
+            print(f"  Op Phase: {a['op_phase']}")
+        if a['op_msg']:
+            print(f"  Op Msg: {a['op_msg']}")
+        if a['conditions']:
+            print(f"  Conditions: {a['conditions']}")
+            
+    print(f"\n=== SYNCED APPS ({len(synced)}) ===")
+    for a in synced:
+        print(f"  {a['name']}: Sync={a['sync']}, Health={a['health']}, NS={a['dest_ns']}")
+
+except Exception as e:
+    print("Error querying Argo CD API:", e)
+
--- a/scripts/check_argocd_resources.py
+++ b/scripts/check_argocd_resources.py
@@ -0,0 +1,29 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
+headers = {
+    "Authorization": f"Bearer {token}",
+    "Content-Type": "application/json"
+}
+
+resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
+data = resp.json()
+items = data.get("items", [])
+
+for app in items:
+    name = app.get("metadata", {}).get("name")
+    sync = app.get("status", {}).get("sync", {}).get("status")
+    health = app.get("status", {}).get("health", {}).get("status")
+    resources = app.get("status", {}).get("resources", [])
+    
+    out_resources = [r for r in resources if r.get("status") != "Synced" or r.get("health", {}).get("status") not in ["Healthy", None]]
+    
+    print(f"[{name}] Sync: {sync}, Health: {health}, Total Resources: {len(resources)}")
+    if out_resources:
+        print(f"  -> Out-of-sync / unhealthy resources:")
+        for r in out_resources:
+            print(f"     {r.get('kind')}/{r.get('name')}: Sync={r.get('status')}, Health={r.get('health', {}).get('status')}")
+
--- a/scripts/check_authentik_proxy_providers.py
+++ b/scripts/check_authentik_proxy_providers.py
@@ -0,0 +1,25 @@
+import urllib.request, ssl, json
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+AK_BASE = "https://authentik.snarfnet.net"
+AK_TOKEN = "SnarfNetPermanentAutomationToken2026"
+
+def req(endpoint):
+    url = f"{AK_BASE}/api/v3/{endpoint}"
+    r = urllib.request.Request(url, headers={"Authorization": f"Bearer {AK_TOKEN}"})
+    with urllib.request.urlopen(r, timeout=10, context=ctx) as res:
+        return json.loads(res.read().decode())
+
+print("=== Authentik Providers (Proxy) ===")
+providers = req("providers/proxy/")
+for p in providers.get("results", []):
+    print(p["pk"], p["name"], p.get("internal_host"), p.get("external_host"), p.get("mode"))
+
+print("\n=== Authentik Outposts ===")
+outposts = req("outposts/instances/")
+for o in outposts.get("results", []):
+    print(o["pk"], o["name"], o["type"], [p for p in o.get("providers", [])])
+
--- a/scripts/check_awx_monitor.py
+++ b/scripts/check_awx_monitor.py
@@ -0,0 +1,16 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, url, port, accepted_statuscodes_json FROM monitor WHERE name LIKE '%AWX%';" """))
+print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM heartbeat WHERE monitor_id IN (SELECT id FROM monitor WHERE name LIKE '%AWX%') ORDER BY id DESC LIMIT 5;" """))
+
+c.close()
+
--- a/scripts/check_cm_json.py
+++ b/scripts/check_cm_json.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, o, _ = c.exec_command('kubectl get cm homepage-2-1744068158 -n infrastructure -o json')
+import json
+cm = json.loads(o.read().decode())
+print("Data keys in ConfigMap:", list(cm.get('data', {}).keys()))
+print("\n--- settings.yaml ---")
+print(cm.get('data', {}).get('settings.yaml'))
+print("\n--- widgets.yaml ---")
+print(cm.get('data', {}).get('widgets.yaml'))
+
+c.close()
+
--- a/scripts/check_dcgm_status.py
+++ b/scripts/check_dcgm_status.py
@@ -0,0 +1,21 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
+    out = stdout.read().decode().strip()
+    return out
+
+time.sleep(5)
+print("Pods status:")
+print(run("kubectl get pods -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o wide"))
+print("Logs from one pod:")
+pod = run("kubectl get pod -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o jsonpath='{.items[0].metadata.name}'")
+print(run(f"kubectl logs -n ai-stack {pod} --tail=20"))
+
+c.close()
+
--- a/scripts/check_deleted_at.py
+++ b/scripts/check_deleted_at.py
@@ -0,0 +1,26 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    res = o.read().decode().strip()
+    if res:
+        print(res)
+    return res
+
+run("""python3 -c "
+import sqlite3
+con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite')
+cur = con.cursor()
+for r in cur.execute('SELECT id, user_id, title, pinned, \"order\", deleted_at FROM items LIMIT 10;').fetchall():
+    print(r)
+" """)
+
+c.close()
+
--- a/scripts/check_deployment_spec.py
+++ b/scripts/check_deployment_spec.py
@@ -0,0 +1,14 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl get deployment grafana -o jsonpath='{.spec.template.spec.containers[0].volumeMounts}'"))
+
+c.close()
+
--- a/scripts/check_direct_apis.py
+++ b/scripts/check_direct_apis.py
@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def test(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    res = o.read().decode().strip()
+    err = e.read().decode().strip()
+    if res:
+        print(res[:300])
+    if err:
+        print("[stderr]", err[:200])
+
+test("curl -m 3 -s -I http://140.44.4.28:448/ || echo 'Timeout 140.44.4.28:448'")
+test("curl -m 3 -s -I http://140.44.4.31:448/ || echo 'Timeout 140.44.4.31:448'")
+test("curl -m 3 -s http://140.44.4.70:8050/api/v1/devices || echo 'Failed PeaNUT'")
+test("curl -m 3 -s -H 'Authorization: token 47c10c54574131c007302b246352f653facb5062' http://140.44.4.62:90/api/v1/user || echo 'Failed Gitea'")
+
+c.close()
--- a/scripts/check_down_monitors.py
+++ b/scripts/check_down_monitors.py
@@ -0,0 +1,41 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+try:
+    r = requests.get("http://140.44.4.28:448/api/status-page/heartbeat/default", verify=False, timeout=10)
+    data = r.json()
+    heartbeats = data.get("heartbeatList", {})
+    down = []
+    up = []
+    
+    # Also fetch status page monitor list to map IDs to names
+    r_sp = requests.get("http://140.44.4.28:448/api/status-page/default", verify=False, timeout=10)
+    sp_data = r_sp.json()
+    
+    id_to_name = {}
+    for group in sp_data.get("publicGroupList", []):
+        for m in group.get("monitorList", []):
+            id_to_name[str(m.get("id"))] = m.get("name")
+            
+    for mid, beats in heartbeats.items():
+        name = id_to_name.get(str(mid), f"Monitor #{mid}")
+        if beats:
+            last = beats[-1]
+            if last.get("status") == 1:
+                up.append((name, last.get("ping")))
+            else:
+                down.append((name, last.get("msg")))
+                
+    print(f"Total Monitors: {len(heartbeats)}")
+    print(f"Up: {len(up)} | Down: {len(down)}")
+    if down:
+        print("\nDown Monitors:")
+        for name, msg in down:
+            print(f"  ❌ {name}: {msg}")
+    else:
+        print("\nAll monitors are UP! 🎉")
+except Exception as e:
+    print("Error:", e)
+
--- a/scripts/check_gaming_pc.py
+++ b/scripts/check_gaming_pc.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Ping 140.44.4.80:")
+print(run("ping -c 2 -W 1 140.44.4.80 || true"))
+print("ARP table for bc:24:11:67:16:cd:")
+print(run("ip neigh | grep -i 'bc:24:11:67:16:cd' || arp -a | grep -i 'bc:24:11:67:16:cd' || true"))
+
+c.close()
+
--- a/scripts/check_glances_host.py
+++ b/scripts/check_glances_host.py
@@ -0,0 +1,19 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+print("ps aux | grep glances:")
+print(run("ps aux | grep glances"))
+
+print("\nnetstat / ss listening:")
+print(run("ss -tulpn | grep 61208 || ss -tulpn | grep glances"))
+
+c.close()
+
--- a/scripts/check_gotify_files.py
+++ b/scripts/check_gotify_files.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("ls -la /mnt/swarm_shared/legacy_volumes/gotify/_data/"))
+
+c.close()
--- a/scripts/check_gotify_idx.py
+++ b/scripts/check_gotify_idx.py
@@ -0,0 +1,22 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_script = """import sqlite3
+conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/gotify.db')
+cur = conn.cursor()
+indexes = cur.execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name='applications'").fetchall()
+for idx in indexes:
+    print(idx)
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/check_idx.py && python3 /tmp/check_idx.py")
+stdin.write(py_script)
+stdin.close()
+print(stdout.read().decode())
+print(stderr.read().decode())
+
+c.close()
+
--- a/scripts/check_gotify_mount.py
+++ b/scripts/check_gotify_mount.py
@@ -0,0 +1,16 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+print("Gotify inspect mounts:")
+print(run("docker service inspect gotify_gotify --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'"))
+
+c.close()
+
--- a/scripts/check_gotify_service.py
+++ b/scripts/check_gotify_service.py
@@ -0,0 +1,19 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+print("Gotify service ps:")
+print(run("docker service ps gotify_gotify --no-trunc | head -n 10"))
+
+print("\nGotify service logs:")
+print(run("docker service logs gotify_gotify --tail=25"))
+
+c.close()
+
--- a/scripts/check_grafana_api.py
+++ b/scripts/check_grafana_api.py
@@ -0,0 +1,27 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+auth = ('admin', 'd0KJLZJXUrbIhW1sjU8lrdFYj7VXts26V7Jw7xQr')
+
+try:
+    r = requests.get('https://grafana.snarfnet.net/api/datasources', auth=auth, verify=False, timeout=10)
+    print("Datasources Status:", r.status_code)
+    if r.status_code == 200:
+        for ds in r.json():
+            print(f"  - ID: {ds.get('id')}, Name: {ds.get('name')}, Type: {ds.get('type')}, URL: {ds.get('url')}, isDefault: {ds.get('isDefault')}")
+except Exception as e:
+    print("Error getting datasources:", e)
+
+try:
+    r = requests.get('https://grafana.snarfnet.net/api/search', auth=auth, verify=False, timeout=10)
+    print("Dashboards Status:", r.status_code)
+    if r.status_code == 200:
+        dashboards = r.json()
+        print(f"Total dashboards: {len(dashboards)}")
+        for d in dashboards[:15]:
+            print(f"  - {d.get('title')} (uri: {d.get('uri')}, type: {d.get('type')})")
+except Exception as e:
+    print("Error getting dashboards:", e)
+
--- a/scripts/check_grafana_dashboards_live.py
+++ b/scripts/check_grafana_dashboards_live.py
@@ -0,0 +1,15 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+auth = ("admin", "Katzir476!!!")
+url = "https://grafana.snarfnet.net/api/search"
+
+resp = requests.get(url, auth=auth, verify=False, timeout=10)
+print("Status:", resp.status_code)
+dashboards = resp.json()
+print(f"Total Dashboards Found: {len(dashboards)}")
+for d in dashboards:
+    print(f" - [{d.get('type')}] Title: {d.get('title')} | UID: {d.get('uid')} | URL: {d.get('url')}")
+
--- a/scripts/check_grafana_diff.py
+++ b/scripts/check_grafana_diff.py
@@ -0,0 +1,14 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl diff -f /tmp/grafana_full.yaml || true"))
+
+c.close()
+
--- a/scripts/check_grafana_info.py
+++ b/scripts/check_grafana_info.py
@@ -0,0 +1,21 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Namespaces on Aerodrome:")
+print(run("kubectl get ns"))
+
+print("Grafana pod:")
+pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
+print(pod)
+print("Grafana admin secret:")
+print(run("kubectl get secret grafana -o jsonpath='{.data.admin-password}' | base64 -d"))
+
+c.close()
+
--- a/scripts/check_grafana_internal.py
+++ b/scripts/check_grafana_internal.py
@@ -0,0 +1,23 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
+print("Pod:", pod)
+print("ConfigMap grafana:")
+print(run("kubectl get cm grafana -o yaml"))
+
+print("Testing local curl inside pod to localhost:3000/api/health:")
+print(run(f"kubectl exec {pod} -- wget -qO- http://localhost:3000/api/health"))
+
+print("Grafana env:")
+print(run(f"kubectl exec {pod} -- env | grep -i grafana"))
+
+c.close()
+
--- a/scripts/check_grafana_load.py
+++ b/scripts/check_grafana_load.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
+print("Grafana pod:", pod)
+print("Grafana provisioning logs:")
+print(run(f"kubectl logs {pod} | grep -i -E 'provision|datasource|dashboard' | tail -n 25"))
+
+c.close()
+
--- a/scripts/check_grafana_mounts.py
+++ b/scripts/check_grafana_mounts.py
@@ -0,0 +1,19 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
+print(run(f"kubectl exec {pod} -- ls -la /etc/grafana/provisioning/datasources"))
+print(run(f"kubectl exec {pod} -- cat /etc/grafana/provisioning/datasources/datasources.yaml"))
+print(run(f"kubectl exec {pod} -- ls -la /etc/grafana/provisioning/dashboards"))
+print(run(f"kubectl exec {pod} -- cat /etc/grafana/provisioning/dashboards/dashboards.yaml"))
+print(run(f"kubectl exec {pod} -- ls -la /var/lib/grafana-dashboards"))
+
+c.close()
+
--- a/scripts/check_grafana_rollout.py
+++ b/scripts/check_grafana_rollout.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl get pods -l app.kubernetes.io/name=grafana"))
+c.close()
+
--- a/scripts/check_grafana_secret.py
+++ b/scripts/check_grafana_secret.py
@@ -0,0 +1,15 @@
+import paramiko
+import base64
+import json
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, o, _ = c.exec_command('kubectl get secret grafana -n default -o json')
+data = json.loads(o.read().decode()).get('data', {})
+for k, v in data.items():
+    print(k, ":", base64.b64decode(v).decode('utf-8', errors='ignore'))
+
+c.close()
+
--- a/scripts/check_grafana_sqlite.py
+++ b/scripts/check_grafana_sqlite.py
@@ -0,0 +1,30 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_script = """import sqlite3
+conn = sqlite3.connect('/tmp/grafana_real.db')
+cur = conn.cursor()
+tables = [t[0] for t in cur.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall()]
+for t in tables:
+    if 'source' in t or 'dash' in t or 'user' in t:
+        print('Matching table:', t)
+
+print('--- data_source content ---')
+for row in cur.execute('SELECT * FROM data_source').fetchall():
+    print(row)
+
+print('--- dashboard content ---')
+for row in cur.execute('SELECT id, title, slug, uid FROM dashboard').fetchall():
+    print(row)
+"""
+
+stdin, stdout, stderr = c.exec_command("cat > /tmp/inspect_grafana.py && python3 /tmp/inspect_grafana.py")
+stdin.write(py_script)
+stdin.close()
+print(stdout.read().decode())
+print(stderr.read().decode())
+
+c.close()
--- a/scripts/check_grafana_success.py
+++ b/scripts/check_grafana_success.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
+print("Grafana pod:", pod)
+print("Provisioning logs:")
+print(run(f"kubectl logs {pod} | grep -i -E 'provision|datasource|dashboard'"))
+
+c.close()
+
--- a/scripts/check_ha_proxy_js.py
+++ b/scripts/check_ha_proxy_js.py
@@ -0,0 +1,31 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
+py_cmd = """
+node -e '
+const fs = require("fs");
+const content = fs.readFileSync("/app/.next/server/pages/api/services/proxy.js", "utf8");
+const idx = content.indexOf("states/");
+if (idx !== -1) {
+    console.log("Found states/ at", idx);
+    console.log(content.slice(Math.max(0, idx - 500), idx + 1000));
+} else {
+    const idx2 = content.indexOf("homeassistant");
+    console.log("Found homeassistant at", idx2);
+    console.log(content.slice(Math.max(0, idx2 - 500), idx2 + 1000));
+}
+'
+"""
+stdin, stdout, stderr = c.exec_command(f'kubectl exec -n infrastructure {pod} -- sh -c \'{py_cmd}\'')
+print(stdout.read().decode())
+print(stderr.read().decode())
+
+c.close()
--- a/scripts/check_harvester_capacity.py
+++ b/scripts/check_harvester_capacity.py
@@ -0,0 +1,25 @@
+import paramiko, json
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; kubectl get nodes -o json'"
+stdin, stdout, stderr = c.exec_command(cmd)
+out = stdout.read().decode()
+data = json.loads(out)
+
+total_cpu = 0
+total_mem_kb = 0
+for n in data['items']:
+    cpu = int(n['status']['capacity']['cpu'])
+    mem_str = n['status']['capacity']['memory']
+    if mem_str.endswith('Ki'):
+        mem_kb = int(mem_str[:-2])
+    total_cpu += cpu
+    total_mem_kb += mem_kb
+
+ram_gib = total_mem_kb / (1024 * 1024)
+print(f"Spacedock Harvester: nodes = {len(data['items'])}, cpu = {total_cpu}, ram = {ram_gib:.1f} GiB")
+c.close()
+
--- a/scripts/check_harvester_prom.py
+++ b/scripts/check_harvester_prom.py
@@ -0,0 +1,21 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
+    out = stdout.read().decode().strip()
+    return out
+
+print("=== Check all in cattle-monitoring-system on Harvester ===")
+print(run("kubectl get all -n cattle-monitoring-system"))
+
+print("\n=== Check CRDs in monitoring ===")
+print(run("kubectl get crd | grep -i monitoring"))
+
+print("\n=== Check Prometheus resources ===")
+print(run("kubectl get prometheus -A"))
+
+c.close()
--- a/scripts/check_harvester_storage.py
+++ b/scripts/check_harvester_storage.py
@@ -0,0 +1,33 @@
+import paramiko, json
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    full_cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; " + cmd + "'"
+    stdin, stdout, stderr = c.exec_command(full_cmd)
+    return stdout.read().decode().strip()
+
+print("=== Blockdevices on Harvester ===")
+print(run("kubectl get blockdevices.harvesterhci.io -A -o wide"))
+
+print("\n=== Longhorn storage on Harvester nodes ===")
+out = run("kubectl get nodes.longhorn.io -n longhorn-system -o json")
+data = json.loads(out)
+total_max = 0
+total_avail = 0
+num_disks = 0
+for n in data['items']:
+    disks = n.get('status', {}).get('diskStatus', {})
+    for d_name, d_val in disks.items():
+        num_disks += 1
+        total_max += d_val.get('storageMaximum', 0)
+        total_avail += d_val.get('storageAvailable', 0)
+
+tib = total_max / (1024**4)
+avail_tib = total_avail / (1024**4)
+print(f"Harvester Disks: {num_disks}, Total Storage: {tib:.2f} TiB, Available: {avail_tib:.2f} TiB")
+
+c.close()
+
--- a/scripts/check_heartbeats.py
+++ b/scripts/check_heartbeats.py
@@ -0,0 +1,20 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+print(f"Pod: {pod}")
+print("Recent heartbeats summary:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e 'USE kuma; SELECT status, count(*) FROM heartbeat GROUP BY status;'"))
+
+print("\nSample recent heartbeats:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e 'USE kuma; SELECT m.name, h.status, h.msg, h.ping FROM heartbeat h JOIN monitor m ON h.monitor_id = m.id ORDER BY h.time DESC LIMIT 15;'"))
+
+c.close()
+
--- a/scripts/check_heimdall_active.py
+++ b/scripts/check_heimdall_active.py
@@ -0,0 +1,31 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    res = o.read().decode().strip()
+    if res:
+        print(res)
+    return res
+
+run("""python3 -c "
+import sqlite3
+con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite')
+cur = con.cursor()
+print('Active items (deleted_at IS NULL):', cur.execute('SELECT count(*) FROM items WHERE deleted_at IS NULL;').fetchone()[0])
+print('Deleted items (deleted_at IS NOT NULL):', cur.execute('SELECT count(*) FROM items WHERE deleted_at IS NOT NULL;').fetchone()[0])
+print('Active items duplicates:')
+print(cur.execute('SELECT count(*), title FROM items WHERE deleted_at IS NULL GROUP BY title HAVING count(*) > 1;').fetchall())
+print('Active pinned items:')
+for r in cur.execute('SELECT id, title, \"order\", pinned FROM items WHERE deleted_at IS NULL AND pinned = 1 ORDER BY \"order\";').fetchall():
+    print(r)
+" """)
+
+c.close()
+
--- a/scripts/check_heimdall_integrity.py
+++ b/scripts/check_heimdall_integrity.py
@@ -0,0 +1,22 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    res = o.read().decode().strip()
+    if res:
+        print(res)
+    return res
+
+run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'PRAGMA integrity_check;'")
+run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1;'")
+run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'SELECT id, title, pinned, \"order\" FROM items ORDER BY \"order\", title LIMIT 30;'")
+
+c.close()
+
--- a/scripts/check_heimdall_pinned.py
+++ b/scripts/check_heimdall_pinned.py
@@ -0,0 +1,27 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    res = o.read().decode().strip()
+    if res:
+        print(res)
+    return res
+
+run("""python3 -c "
+import sqlite3
+con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite')
+cur = con.cursor()
+print('Active items breakdown:')
+for r in cur.execute('SELECT id, user_id, title, url, pinned, \"order\", deleted_at FROM items WHERE deleted_at IS NULL LIMIT 20;').fetchall():
+    print(r)
+" """)
+
+c.close()
+
--- a/scripts/check_heimdall_python.py
+++ b/scripts/check_heimdall_python.py
@@ -0,0 +1,24 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f"\n>>> {cmd}")
+    _, o, e = c.exec_command(cmd)
+    res = o.read().decode().strip()
+    err = e.read().decode().strip()
+    if res:
+        print(res)
+    if err:
+        print("[stderr]", err)
+    return res
+
+run("which sqlite3 || echo 'sqlite3 not installed on host'")
+run("python3 -c \"import sqlite3; con=sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite'); cur=con.cursor(); print(cur.execute('PRAGMA integrity_check;').fetchall()); print('Total items:', cur.execute('SELECT count(*) FROM items;').fetchone()[0]); print('Duplicates:', cur.execute('SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1;').fetchall())\"")
+
+c.close()
+
--- a/scripts/check_homepage_api.py
+++ b/scripts/check_homepage_api.py
@@ -0,0 +1,26 @@
+import requests
+import json
+import urllib3
+urllib3.disable_warnings()
+
+s = requests.Session()
+r = s.get('https://homepage.snarfnet.net/api/services', verify=False)
+print("Status /api/services:", r.status_code)
+try:
+    data = r.json()
+    print("Services JSON keys/length:", len(data))
+    for group in data:
+        for title, items in group.items():
+            print(f"Group: {title}")
+            for item in items:
+                print(f"  - {item.get('name')}: widget={item.get('widget')}")
+except Exception as e:
+    print("Error:", e, r.text[:200])
+
+r2 = s.get('https://homepage.snarfnet.net/api/widgets', verify=False)
+print("Status /api/widgets:", r2.status_code)
+try:
+    print("Widgets:", json.dumps(r2.json(), indent=2))
+except Exception as e:
+    print("Error widgets:", e, r2.text[:200])
+
--- a/scripts/check_homepage_html.py
+++ b/scripts/check_homepage_html.py
@@ -0,0 +1,16 @@
+import urllib.request, ssl, re
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+req = urllib.request.Request('https://homepage.snarfnet.net/', headers={'User-Agent': 'Mozilla/5.0'})
+resp = urllib.request.urlopen(req, context=ctx)
+html = resp.read().decode('utf-8')
+
+for pat in [r'href="([^"]+)"', r'src="([^"]+)"']:
+    matches = re.findall(pat, html)
+    for m in matches:
+        if any(k in m for k in ['argo', 'tempest', 'weather', 'icon', 'png', 'svg']):
+            print(m)
+
--- a/scripts/check_homepage_info.py
+++ b/scripts/check_homepage_info.py
@@ -0,0 +1,11 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+_, o, _ = c.exec_command('kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath="{.spec.template.spec.containers[0].image}"')
+print("Image:", o.read().decode())
+_, o, _ = c.exec_command('kubectl logs deployment/homepage-2-1744068158 -n infrastructure --tail=20')
+print("Logs:\n", o.read().decode())
+c.close()
+
--- a/scripts/check_homepage_pod_logs.py
+++ b/scripts/check_homepage_pod_logs.py
@@ -0,0 +1,16 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
+print("Logs from Homepage pod:")
+print(run(f"kubectl logs -n infrastructure {pod} --tail=30"))
+
+c.close()
+
--- a/scripts/check_homepage_rbac.py
+++ b/scripts/check_homepage_rbac.py
@@ -0,0 +1,21 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
+print("Pod:", pod)
+
+print("Check can-i list nodes:")
+print(run(f"kubectl auth can-i list nodes --as=system:serviceaccount:infrastructure:default"))
+print("Check can-i get nodes.metrics.k8s.io:")
+print(run(f"kubectl auth can-i get nodes.metrics.k8s.io --as=system:serviceaccount:infrastructure:default"))
+
+c.close()
+
--- a/scripts/check_homepage_routes.py
+++ b/scripts/check_homepage_routes.py
@@ -0,0 +1,15 @@
+import requests
+import urllib3
+urllib3.disable_warnings()
+
+s = requests.Session()
+r = s.get('https://homepage.snarfnet.net', verify=False)
+print("Page length:", len(r.text))
+import re
+apis = re.findall(r'/api/[a-zA-Z0-9_/]+', r.text)
+print("Found API routes:", set(apis))
+
+# Also test hitting the Home Assistant custom widget api if any
+# In Homepage, the widget endpoint is /api/widgets/homeassistant or similar or /api/services/...
+# Let's check pod files to see what routes exist
+
--- a/scripts/check_icon_resolver.py
+++ b/scripts/check_icon_resolver.py
@@ -0,0 +1,18 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'")
+
+# Search for github raw or cdn in server.js or .next
+print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'cdn.jsdelivr.net' /app/.next/server/ | head -n 5"))
+print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'dashboard-icons' /app/src/"))
+
+c.close()
+
--- a/scripts/check_kuma_db_backups.py
+++ b/scripts/check_kuma_db_backups.py
@@ -0,0 +1,21 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+print("Pod:", pod)
+print("Check for any .sql or .db files in /app/data:")
+print(run(f"kubectl exec -n monitoring {pod} -- find /app/data -name '*.sql*' -o -name '*.db*' -o -name '*.bak*'"))
+
+print("\nCheck Notification in DB:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT id, name, type, is_default FROM notification;'"))
+
+c.close()
+
--- a/scripts/check_kuma_details.py
+++ b/scripts/check_kuma_details.py
@@ -0,0 +1,23 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+
+print("Monitors count:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT count(id) FROM monitor;'"))
+
+print("\nStatus page count:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT * FROM status_page;'"))
+
+print("\nGroup count:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT * FROM `group`;'"))
+
+c.close()
+
--- a/scripts/check_kuma_live_status.py
+++ b/scripts/check_kuma_live_status.py
@@ -0,0 +1,27 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("=== Uptime Kuma Pods ===")
+print(run("kubectl get pods -n monitoring -o wide"))
+
+print("\n=== Uptime Kuma Deployment & Volumes ===")
+print(run("kubectl get deployment -n monitoring uptime-kuma-2-1756668961 -o yaml | grep -A 10 volumes"))
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma-2-1756668961 -o jsonpath='{.items[0].metadata.name}'")
+print(f"\n=== Current Pod: {pod} ===")
+
+print("\n=== Query MariaDB monitors table ===")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT count(*) FROM monitor; SELECT id, name, type, active FROM monitor;'"))
+
+print("\n=== Files in /app/data ===")
+print(run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data"))
+
+c.close()
+
--- a/scripts/check_kuma_live_status2.py
+++ b/scripts/check_kuma_live_status2.py
@@ -0,0 +1,25 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    err = e.read().decode().strip()
+    return out + ("\nERR: " + err if err else "")
+
+pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64"
+print(f"=== Querying Pod: {pod} ===")
+print("Query MariaDB monitor count:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT count(*) FROM monitor;'"))
+
+print("Query all monitors:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT id, name, type, active FROM monitor;'"))
+
+print("\nFiles in /app/data:")
+print(run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data"))
+
+c.close()
+
--- a/scripts/check_kuma_logs.py
+++ b/scripts/check_kuma_logs.py
@@ -0,0 +1,20 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64"
+print("=== Error log from /app/data/error.log ===")
+print(run(f"kubectl exec -n monitoring {pod} -- tail -n 50 /app/data/error.log"))
+
+print("\n=== Recent pod logs ===")
+print(run(f"kubectl logs -n monitoring {pod} --tail=60"))
+
+c.close()
+
--- a/scripts/check_kuma_mariadb.py
+++ b/scripts/check_kuma_mariadb.py
@@ -0,0 +1,17 @@
+import paramiko
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-6656d4d5c6-z5cz9 -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT count(*) AS monitor_count FROM monitor; SELECT id, name, type, active FROM monitor LIMIT 15; SELECT * FROM notification;" """
+_, stdout, stderr = c.exec_command(cmd)
+print(stdout.read().decode())
+err = stderr.read().decode()
+if err:
+    print("[stderr]", err)
+
+c.close()
+
--- a/scripts/check_kuma_notif.py
+++ b/scripts/check_kuma_notif.py
@@ -0,0 +1,16 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT * FROM notification;'"))
+
+c.close()
+
--- a/scripts/check_kuma_slugs.py
+++ b/scripts/check_kuma_slugs.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+print("group:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'DESCRIBE `group`;'"))
+print("\nmonitor_group:")
+print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'DESCRIBE monitor_group;'"))
+
+c.close()
--- a/scripts/check_kuma_sp_schema.py
+++ b/scripts/check_kuma_sp_schema.py
@@ -0,0 +1,20 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
+
+print("--- STATUS PAGE SCHEMA ---")
+print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE status_page; DESCRIBE \`group\`; DESCRIBE monitor_group;" """))
+
+print("\n--- CURRENT GROUPS ---")
+print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM \`group\`;" """))
+
+c.close()
+
--- a/scripts/check_live_cm.py
+++ b/scripts/check_live_cm.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+_, o, _ = c.exec_command('kubectl get cm homepage-2-1744068158 -n infrastructure -o yaml')
+cm_yaml = o.read().decode()
+start = cm_yaml.find('settings.yaml:')
+print(cm_yaml[start:start+400])
+
+c.close()
+
--- a/scripts/check_longhorn_source.py
+++ b/scripts/check_longhorn_source.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
+print('Pod:', pod)
+print('Search for longhorn in /app/src or /app:')
+print(run(f'kubectl exec -n infrastructure {pod} -- grep -rn "Missing Longhorn URL" /app'))
+
+c.close()
+
--- a/scripts/check_longhorn_svc.py
+++ b/scripts/check_longhorn_svc.py
@@ -0,0 +1,19 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    out = o.read().decode().strip()
+    return out
+
+print("Spaceport longhorn-system services:")
+print(run("kubectl get svc -n longhorn-system"))
+
+print("\nSpaceport ingress:")
+print(run("kubectl get ingress -n longhorn-system"))
+
+c.close()
+
--- a/scripts/check_mdi_icons.py
+++ b/scripts/check_mdi_icons.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'")
+
+print("Checking mdi weather in @mdi/js:")
+print(run(f"kubectl exec -n infrastructure {pod} -- node -e \"const mdi = require('@mdi/js'); console.log('mdiWeatherStation:', !!mdi.mdiWeatherStation); console.log('mdiWeatherPartlyCloudy:', !!mdi.mdiWeatherPartlyCloudy);\""))
+
+c.close()
+
--- a/scripts/check_monitoring_ns.py
+++ b/scripts/check_monitoring_ns.py
@@ -0,0 +1,17 @@
+import paramiko
+
+def check_ns(ip, name):
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    print(f"=== {name} ({ip}) ===")
+    _, o, _ = c.exec_command("kubectl get all -n monitoring")
+    print(o.read().decode())
+    if name == 'Aerodrome':
+        _, o, _ = c.exec_command("kubectl get all -n cattle-monitoring-system")
+        print("cattle-monitoring-system:\n", o.read().decode())
+    c.close()
+
+check_ns('140.44.4.11', 'Aerodrome')
+check_ns('140.44.4.41', 'Spaceport')
+
--- a/scripts/check_monitoring_spaceport.py
+++ b/scripts/check_monitoring_spaceport.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl get all -n monitoring"))
+c.close()
+
--- a/scripts/check_nas_backups.py
+++ b/scripts/check_nas_backups.py
@@ -0,0 +1,18 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Check /mnt/nfs_shares/backups:")
+print(run("ls -la /mnt/nfs_shares/backups"))
+
+print("Create /mnt/nfs_shares/game-backups:")
+print(run("mkdir -p /mnt/nfs_shares/game-backups && ls -la /mnt/nfs_shares/game-backups"))
+
+c.close()
+
--- a/scripts/check_nas_ports.py
+++ b/scripts/check_nas_ports.py
@@ -0,0 +1,12 @@
+import socket
+
+for ip in ['14.10.10.71', '14.10.10.72']:
+    print(f"\nChecking ports on {ip}:")
+    for port in [22, 80, 111, 161, 443, 445, 2049, 5000, 5001]:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(1.0)
+        res = s.connect_ex((ip, port))
+        if res == 0:
+            print(f"  Port {port} OPEN")
+        s.close()
+
--- a/scripts/check_nat.py
+++ b/scripts/check_nat.py
@@ -0,0 +1,20 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    print(f">>> {cmd}")
+    _, o, e = ssh.exec_command(cmd)
+    print(o.read().decode().strip())
+    err = e.read().decode().strip()
+    if err:
+        print("[stderr]", err)
+
+run("iptables -t nat -L PREROUTING -n -v")
+run("iptables -t nat -L POSTROUTING -n -v")
+run("sysctl net.ipv4.ip_forward")
+
+ssh.close()
+
--- a/scripts/check_npm_db.py
+++ b/scripts/check_npm_db.py
@@ -2,14 +2,14 @@ import paramiko

 ssh = paramiko.SSHClient()
 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)

-cmd = "sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c \"python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print('\\n'.join(str(r) for r in cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE is_deleted=0')))\\\"\""
-stdin, stdout, stderr = ssh.exec_command(cmd)
+stdin, stdout, stderr = ssh.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') cat /app/templates/proxy_host.conf")
+print("--- TEMPLATE ---")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/21.conf")
+print("--- 21.CONF ---")
 print(stdout.read().decode())
-err = stderr.read().decode()
-if err:
-    print("ERR:", err)

 ssh.close()
-
--- a/scripts/check_npm_ports.py
+++ b/scripts/check_npm_ports.py
@@ -0,0 +1,20 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("docker ps --filter 'name=nginx-proxy-manager_app' --format '{{.Ports}}'")
+print("NPM Container Ports:")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("docker inspect $(docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{json .HostConfig.PortBindings}}'")
+print("NPM HostConfig PortBindings:")
+print(stdout.read().decode())
+
+stdin, stdout, stderr = ssh.exec_command("docker inspect $(docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{.HostConfig.NetworkMode}}'")
+print("NPM NetworkMode:")
+print(stdout.read().decode())
+
+ssh.close()
+
--- a/scripts/check_npm_streams.py
+++ b/scripts/check_npm_streams.py
@@ -0,0 +1,11 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+stdin, stdout, stderr = ssh.exec_command("python3 -c \"import sqlite3; con = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'); print(list(con.execute('SELECT id, incoming_port, forwarding_host, forwarding_port, is_deleted FROM stream')))\"")
+print("STREAMS:", stdout.read().decode())
+print("ERR:", stderr.read().decode())
+
+ssh.close()
--- a/scripts/check_nvidia_host.py
+++ b/scripts/check_nvidia_host.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
+    return stdout.read().decode().strip()
+
+print("Check nvidia-smi:")
+print(run("which nvidia-smi || nvidia-smi"))
+print("Check GPU devices:")
+print(run("ls -l /dev/nvidia*"))
+
+c.close()
+
--- a/scripts/check_nvidia_output.py
+++ b/scripts/check_nvidia_output.py
@@ -0,0 +1,15 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
+    return stdout.read().decode().strip()
+
+print("nvidia-smi output:")
+print(run("/usr/local/bin/nvidia-smi"))
+
+c.close()
+
--- a/scripts/check_one_problem.py
+++ b/scripts/check_one_problem.py
@@ -0,0 +1,24 @@
+import json, ssl, urllib.request
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
+
+payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
+token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+payload = {'jsonrpc': '2.0', 'method': 'problem.get', 'params': {'output': 'extend'}, 'id': 2}
+req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
+problems = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
+
+for p in problems:
+    if 'active checks' in p['name'].lower():
+        # Get event
+        ep = {'jsonrpc': '2.0', 'method': 'event.get', 'params': {'eventids': [p['eventid']], 'selectHosts': ['host', 'name']}, 'id': 3}
+        er = urllib.request.Request(ZABBIX_URL, data=json.dumps(ep).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
+        events = json.loads(urllib.request.urlopen(er, context=ctx).read())['result']
+        print(f"Problem: {p['name']} | Event: {events}")
+
--- a/scripts/check_pkg_icons.py
+++ b/scripts/check_pkg_icons.py
@@ -0,0 +1,17 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'")
+
+print("Checking package.json dependencies:")
+print(run(f"kubectl exec -n infrastructure {pod} -- grep -E 'mdi|icon|simple' /app/package.json"))
+
+c.close()
+
--- a/scripts/check_pod_logs.py
+++ b/scripts/check_pod_logs.py
@@ -0,0 +1,16 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
+print('Pod:', pod)
+print('Recent pod logs:')
+print(run(f'kubectl logs -n infrastructure {pod} --tail=50'))
+c.close()
+
--- a/scripts/check_ports.py
+++ b/scripts/check_ports.py
@@ -0,0 +1,24 @@
+import socket
+
+hosts = [
+    ('140.44.4.11', 9100, 'Aerodrome Master 01'),
+    ('140.44.4.41', 9100, 'Spaceport Master 01'),
+    ('140.44.4.71', 9100, 'Shipyard Swarm'),
+    ('140.44.4.81', 9100, 'Harvester Node 01'),
+    ('140.44.4.81', 9400, 'Harvester Node 01 DCGM'),
+    ('140.44.4.11', 10250, 'Aerodrome Kubelet'),
+    ('140.44.4.41', 10250, 'Spaceport Kubelet'),
+    ('140.44.4.60', 8888, 'Zabbix Web'),
+    ('140.44.4.64', 10051, 'Zabbix Server'),
+]
+
+for ip, port, desc in hosts:
+    s = socket.socket()
+    s.settimeout(1.5)
+    try:
+        s.connect((ip, port))
+        print(f"[OPEN] {ip}:{port} ({desc})")
+        s.close()
+    except Exception as e:
+        print(f"[CLOSED/ERR] {ip}:{port} ({desc}) - {e}")
+
--- a/scripts/check_prom_elsewhere.py
+++ b/scripts/check_prom_elsewhere.py
@@ -0,0 +1,14 @@
+import paramiko
+
+def check_host(ip, name, cmd):
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    print(f"=== {name} ({ip}) ===")
+    _, o, _ = c.exec_command(cmd)
+    print(o.read().decode())
+    c.close()
+
+check_host('140.44.4.71', 'Swarm', 'docker service ls; docker ps | grep -i prom')
+check_host('140.44.44.121', 'Spacedock', 'kubectl get pods,svc -A | grep -i prom')
+
--- a/scripts/check_prom_live_targets.py
+++ b/scripts/check_prom_live_targets.py
@@ -0,0 +1,29 @@
+import paramiko
+import json
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+out = run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets")
+data = json.loads(out)
+print("=== ACTIVE TARGETS ===")
+for t in data.get('data', {}).get('activeTargets', []):
+    print(f"[{t.get('health')}] pool: {t.get('scrapePool')} | job: {t.get('labels', {}).get('job')} | instance: {t.get('labels', {}).get('instance')} | url: {t.get('scrapeUrl')}")
+    if t.get('lastError'):
+        print(f"    ERROR: {t.get('lastError')}")
+
+print("\n=== SAMPLE NODE EXPORTER QUERY ===")
+out = run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(node_cpu_seconds_total{mode=\"idle\"})_by_(instance)'")
+print(out)
+
+print("\n=== SAMPLE CADVISOR QUERY ===")
+out = run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(container_cpu_usage_seconds_total)'")
+print(out)
+
+c.close()
+
--- a/scripts/check_prom_metrics.py
+++ b/scripts/check_prom_metrics.py
@@ -0,0 +1,21 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("--- PROMETHEUS TARGETS ---")
+print(run("curl -s http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | \"\\(.labels.job) -> \\(.scrapeUrl) (\\(.health))\"' | head -n 30"))
+
+print("\n--- SAMPLE NODE METRICS ---")
+print(run("curl -s 'http://localhost:9090/api/v1/query?query=node_cpu_seconds_total' | jq -r '.data.result | length'"))
+
+print("\n--- SAMPLE CONTAINER METRICS ---")
+print(run("curl -s 'http://localhost:9090/api/v1/query?query=container_cpu_usage_seconds_total' | jq -r '.data.result | length'"))
+
+c.close()
+
--- a/scripts/check_prom_pods.py
+++ b/scripts/check_prom_pods.py
@@ -0,0 +1,13 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print(run("kubectl get pods -A | grep -E 'prom|grafana|node-exporter|cadvisor'"))
+c.close()
+
--- a/scripts/check_prom_sa.py
+++ b/scripts/check_prom_sa.py
@@ -0,0 +1,18 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Prometheus Deployment / ServiceAccount:")
+print(run("kubectl get deployment prometheus -o yaml | grep -E 'serviceAccount|image'"))
+
+print("\nClusterRoles / RoleBindings for prometheus:")
+print(run("kubectl get clusterrolebindings | grep -i prom"))
+
+c.close()
+
--- a/scripts/check_prom_targets.py
+++ b/scripts/check_prom_targets.py
@@ -0,0 +1,19 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+def run(cmd):
+    _, o, e = c.exec_command(cmd)
+    return o.read().decode().strip()
+
+print("Prometheus pod status:")
+print(run("kubectl get pods -l app.kubernetes.io/name=prometheus"))
+
+print("\nPrometheus targets:")
+print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets"))
+
+c.close()
+
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Snarf	336b7759d0	feat: add infrastructure automation and diagnostic scripts for Zabbix, Nginx, and Authentik	2026-11-12 20:18:37 -05:00
Snarf	68c108cfe8	docs(topology): regenerate high-res PNG diagram to reflect latest layout, hardware and storage metrics	2026-11-12 17:13:51 -05:00
Snarf	9ebd48af0c	chore: update homelab-gitops-repo submodule reference	2026-11-12 17:09:51 -05:00
Snarf	113ebdc7c5	feat: add automation and inspection scripts for Nginx Proxy Manager and Authentik integration	2026-11-12 17:08:14 -05:00
Snarf	5a70460e89	feat: add homelab deployment configs, documentation, and automation scripts	2026-11-12 16:57:08 -05:00
Snarf	1228b957de	chore: remove empty file	2026-11-08 15:03:27 -05:00