diff --git a/docker-deployments/heimdall.yml b/docker-deployments/heimdall.yml index 95dd37f..71e10d1 100644 --- a/docker-deployments/heimdall.yml +++ b/docker-deployments/heimdall.yml @@ -1,20 +1,33 @@ +version: "3.8" + services: heimdall: image: lscr.io/linuxserver/heimdall:latest ports: - - "85:80" - - "403:443" + - target: 80 + published: 85 + mode: ingress + - target: 443 + published: 403 + mode: ingress environment: - PUID=1000 - PGID=1000 - TZ=America/New_York volumes: - - heimdall_config:/config + - type: bind + source: /var/lib/docker-data/heimdall/config + target: /config + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - node.hostname == snshpd4lnxswm01 + restart_policy: + condition: on-failure -volumes: - heimdall_config: - driver: local - driver_opts: - type: nfs - o: "addr=14.10.10.71,rw,nfsvers=4" - device: ":/volume1/docker/heimdall/" \ No newline at end of file +networks: + default: + name: homelab_public + external: true diff --git a/docker-deployments/shepherd.yml b/docker-deployments/shepherd.yml new file mode 100644 index 0000000..426ed39 --- /dev/null +++ b/docker-deployments/shepherd.yml @@ -0,0 +1,21 @@ +version: "3.8" + +services: + shepherd: + image: containrrr/shepherd:latest + environment: + - SLEEP_TIME=24h + - IGNORELIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup + - UPDATE_OPTIONS=--with-registry-auth + - ROLLBACK_FAILURE=true + - TZ=America/New_York + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - node.role == manager + restart_policy: + condition: on-failure diff --git a/docs/snarfnet_homelab_topology.html b/docs/snarfnet_homelab_topology.html new file mode 100644 index 0000000..fbccd05 --- /dev/null +++ b/docs/snarfnet_homelab_topology.html @@ -0,0 +1,1029 @@ + + + + + + SnarfNet Homelab Architecture & Topology + + + + + + +
+
+
+ šŸŖ +

+ SnarfNet | Infrastructure & Topology +

+ + ALL CLUSTERS ACTIVE + +
+

+ Complete operational schematic mapping dual-WAN routing, VLAN segmentation, 4 compute clusters (Aerodrome, Spaceport, Spacedock, Shipyard), AI inference fabrics, and storage layers. +

+
+ + +
+
+ + āŒ˜K +
+ +
+
Topology Release
+
Phase 8 ā€¢ Nov 2026
+
+
+
+ + +
+ + +
+ + +
+
+
+

+ + Network VLAN Fabric +

+ 8 Zones +
+ +
+
+
+ VLAN 1 LAN / Core +
UDM-SE Gateway, Core Switches
+
+ 14.44.4.0/25 +
+ +
+
+ VLAN 2 Management +
APs, USP RPS, iDRAC, Switches
+
+ 140.44.44.0/24 +
+ +
+
+ VLAN 20 Compute Fleet +
K3s Aerodrome, Spaceport, Swarm
+
+ 140.44.4.0/25 +
+ +
+
+ VLAN 3 Spacedock Lab +
Harvester HCI Virtualization
+
+ 140.44.1.0/24 +
+ +
+
+ VLAN 40 Storage SAN +
SnarfNAS (DS416play), SN-NAS (RS810+)
+
+ 14.10.10.0/25 +
+ +
+
+ VLAN 50 Smart Home +
Home Assistant, NUT, Tempest
+
+ 14.4.4.0/25 +
+ +
+
+ VLAN 60 HA Interconnect +
Keepalived VIPs & Sync Heartbeats
+
+ 140.4.4.0/25 +
+ +
+
+ VLAN 30 Media Clients +
Smart TVs, Apple TVs, Streaming
+
+ 192.168.1.0/25 +
+
+
+ + +
+

Topology Key

+
+
+ Core Backbone +
+
+ K3s Compute +
+
+ Harvester HCI +
+
+ Swarm Fleet +
+
+ Cloudflare / Tunnel +
+
+ High Availability +
+
+
+
+ + +
+ + +
+
+ +
+
Public Internet
+
Primary WAN: 174.106.252.96
+
+
+ + +
+ +
+
+ WAN 1: Fiber ONT +
+
1 Gbps Symmetric
+
10G SFP+ to UDM-SE
+
+ + +
+
+ WAN 2: 5G Backup +
+
Cellular Standby
+
GbE RJ45 Failover
+
+
+ + +
+ +
+
+ + +
+
+ +
+
+
+ šŸ›”ļø +
+
+

+ UniFi Dream Machine Special Edition (UDM-SE) +

+
Gateway VIP: 14.44.4.1 ā€¢ UniFi OS v4.1 ā€¢ WireGuard Server
+
+
+
+ + 10G SFP+ Backbone + +
+
+ + +
+
+
+ āš” Routing & QoS +
+
    +
  • ā€¢ Inter-VLAN Wire-Speed L3
    • +
  • ā€¢ Smart Queues & FQ-CoDel
    • +
  • ā€¢ VLAN 20 Compute Priority
    • +
+
+ +
+
+ šŸ”’ Edge Security +
+
    +
  • ā€¢ Suricata IPS/IDS Level 5
    • +
  • ā€¢ Deep Packet Inspection
    • +
  • ā€¢ Strict LAN/IoT Isolation
    • +
+
+ +
+
+ šŸŽÆ Core Port Forwards +
+
    +
  • ā€¢ 80/443 ā†’ 140.44.4.74 (NPM Host)
    • +
  • ā€¢ 10051/52 ā†’ 140.44.4.51 (Zabbix)
    • +
  • ā€¢ 8443/27015 ā†’ 14.4.4.15 (PC)
    • +
+
+
+
+ + +
+
+
šŸ”€
+
+
USW Enterprise Core Switches
+
2.5G PoE+ to APs, 10G SFP+ to Racks
+
+
+ +
+
šŸ“¶
+
+
UniFi Wi-Fi 6 Enterprise Fleet
+
SSIDs: SnarfNet-IoT, SnarfNet-5G, Trusted
+
+
+
+ +
+ + +
+ + +
+

+ + Edge Access & Ingress +

+ +
+
+
+ Cloudflare Zero Trust + cloudflared +
+

Outbound-only secure tunnel routes *.snarfnet.net without opening inbound WAN router ports.

+
+ +
+
+ Tailscale Mesh VPN + 100.64.0.0/10 +
+

Subnet routers provide direct wireguard access to all 8 VLANs from mobile devices.

+
+ +
+
+ Authentik Conditional MFA + SSO & Auth +
+

Client IP evaluation: Internal VLANs bypass MFA for frictionless use; external WAN enforces Passkeys/MFA.

+
+
+
+ + +
+
+

+ šŸ§  AI Inference Pipeline +

+ Tesla T4s + Hailo +
+ +
+
+ 1. Frontend + Open WebUI (:8080) +
+
ā†“
+
+ 2. Reasoning + vLLM DeepSeek-R1 +
+
ā†“
+
+ 3. Execution + Triton + Qdrant +
+
+
+ +
+ +
+ + +
+
+
+

+ Compute Clusters & Workload Fleet +

+

All compute clusters operate on VLAN 20 (140.44.4.0/25) with dedicated VIPs and Longhorn block fabrics.

+
+
+ 29 Physical Nodes + 400+ Containers +
+
+ + +
+ + +
+
+
+
+
+ +

Cluster A: Aerodrome

+ Rancher Local +
+
+ 10 Nodes ā€¢ 3 Control Plane + 7 Workers ā€¢ Debian 12 (arm64) ā€¢ MetalLB: 140.44.4.28-39 +
+
+
+
40 CPU Cores
+
65.5 GiB RAM Total
+
+
+ + +
+
+ ctl01-03: 140.44.4.11-13 +
+
+ wrk01-04: 140.44.4.21-24 +
+
+ wrk05-07: 140.44.4.25-27 +
+
+
Storage Type: Longhorn
+
Storage Total: 818 GiB
+
+
+ + +
+
Active Workloads & VIPs
+
+ +
+
+
+ šŸ„ Rancher Management +
+
aerodrome.snarfnet.net
+
+ :443 +
+ +
+
+
+ šŸ”‘ Authentik SSO Portal +
+
authentik.snarfnet.net
+
+ VIP .29 +
+ +
+
+
+ šŸ“Š Grafana Telemetry +
+
grafana.snarfnet.net
+
+ VIP .32 +
+ +
+
+
+ šŸ“ˆ Prometheus TSDB +
+
39 Targets (cAdvisor + Nodes)
+
+ :9090 +
+ +
+
+
+ šŸŸ¢ Uptime Kuma Probes +
+
uptime.snarfnet.net (35 Mon.)
+
+ VIP .28 +
+ +
+
+
+ šŸ¦… AWX Ansible Controller +
+
awx.snarfnet.net
+
+ VIP .30 +
+ +
+
+
+
+ + +
+
+
+
+
+ +

Cluster B: Spaceport

+ GitOps Core +
+
+ 10 Nodes ā€¢ 4 Control Plane + 6 Workers ā€¢ RPi 5 (arm64) ā€¢ MetalLB: 140.44.4.57-65 +
+
+
+
40 CPU Cores
+
54.0 GiB RAM Total
+
+
+ + +
+
+ ctl01-04: 140.44.4.41-44 +
+
+ wrk01-03: 140.44.4.51-53 +
+
+ wrk04-06: 140.44.4.54-56 +
+
+
Storage Type: Longhorn
+
Storage Total: 1.14 TiB
+
+
+ + +
+
Active Workloads & VIPs
+
+ +
+
+
+ šŸ™ ArgoCD Reconciler +
+
argo.snarfnet.net (15 Apps)
+
+ Node .51 +
+ +
+
+
+ šŸµ Gitea Repositories +
+
git.snarfnet.net
+
+ VIP .62 +
+ +
+
+
+ šŸ  Homepage Dashboard +
+
homepage.snarfnet.net
+
+ VIP .60 +
+ +
+
+
+ šŸ›”ļø Zabbix Core Server +
+
zabbix.snarfnet.net (28 Hosts)
+
+ VIP .61 +
+ +
+
+
+ āš™ļø Semaphore Ansible +
+
semaphore.snarfnet.net
+
+ VIP .58 +
+ +
+
+
+ šŸ‘Øā€šŸ’» Coder Dev Cloud +
+
coder.snarfnet.net
+
+ VIP .63 +
+ +
+
+
+
+ + +
+
+
+
+
+ +

Cluster C: Spacedock (Harvester HCI)

+ GPU Inference +
+
+ 4 Physical Nodes ā€¢ AMD64 Server Architecture ā€¢ Dedicated Tesla T4 AI Compute +
+
+
+
224 CPU Cores
+
503 GiB RAM Total ā€¢ 4x Tesla T4 GPUs
+
+
+ + +
+
+ Node 01: 140.44.4.81 +
+
+ Node 02: 140.44.4.82 +
+
+ Node 03: 140.44.4.83 +
+
+ Node 04: 140.44.4.84 +
+
+ + +
+
Active Workloads & Inference Services
+
+ +
+
+
+ šŸŒ¾ Harvester HCI Console +
+
harvester.spacedock.snarfnet.net
+
+ VLAN 3 +
+ +
+
+
+ šŸ§  vLLM DeepSeek-R1-14B +
+
Reasoning Planner (.81)
+
+ :30805 +
+ +
+
+
+ šŸ’Ž vLLM Gemma-4-12B +
+
Coding Specialist (.82)
+
+ :30804 +
+ +
+
+
+ šŸ¤– vLLM Qwen 2.5-7B +
+
Instruction Following (.84)
+
+ :30800 +
+ +
+
+
+ šŸ”± Triton Inference Server +
+
Decision Engine (.83)
+
+ :30803 +
+ +
+
+
+ āš” Qdrant Vector Engine +
+
Semantic Vector Store (.82)
+
+ :30633 +
+ +
+
+
+
+ + +
+
+
+
+
+ +

Cluster D: Shipyard (Docker Swarm)

+ Swarm VIP: 140.44.4.70 +
+
+ 5 RPi 5 Nodes ā€¢ Keepalived VRRP VIP ā€¢ Host-Mode Ingress ā€¢ Portainer EE 2.45 +
+
+
+
20 CPU Cores
+
20.0 GiB RAM Total
+
+
+ + +
+
+ Swm01: .71 +
+
+ Swm02: .72 +
+
+ Swm03: .73 +
+
+ NPM: .74 +
+
+ Swm05: .75 +
+
+ + +
+
Active Workloads & Stacks
+
+ +
+
+
+ šŸš¢ Portainer Enterprise +
+
shipyard.snarfnet.net
+
+ :9443 +
+ +
+
+
+ šŸŒ NGINX Proxy Manager +
+
Host Mode Ingress (.74)
+
+ :80/:443 +
+ +
+
+
+ šŸ”” Gotify Notifications +
+
gotify.snarfnet.net
+
+ :8090 +
+ +
+
+
+ šŸ§­ Heimdall Landing +
+
portal.snarfnet.net
+
+ :85 +
+ +
+
+
+ šŸ’» MeshCentral Remote +
+
mesh.snarfnet.net
+
+ :8086 +
+ +
+
+
+ šŸ“¦ Homebox Inventory +
+
homebox.snarfnet.net
+
+ :3100 +
+ +
+
+
+
+ +
+
+ + +
+ + +
+
+
+ šŸ’¾ +

Storage SAN & Distributed Volumes (VLAN 40)

+
+ 14.10.10.0/25 +
+ +
+ +
+
+
+ SnarfNAS (Synology DS416play) + 14.10.10.72 +
+

Synology DiskStation DS416play desktop appliance, primary NFS/SMB target, media libraries, Time Machine, and Hyper Backup replication.

+
+
+
Drives: 4 Drives
+
Total Storage: 16 TiB Total
+
+
+ + +
+
+
+ SN-NAS (Synology RS810+) + 14.10.10.71 +
+

Synology RackStation RS810+ 1U rackmount appliance, dedicated Btrfs backup pool, iSCSI targets, and Snapshot Replication.

+
+
+
Drives: 4 Drives
+
Total Storage: 37 TiB Total
+
+
+ + +
+
+
+ Longhorn Spaceport Storage + 140.44.4.41-56 +
+

Replicated distributed block storage across Spaceport workers backing Gitea, Semaphore, Jenkins, and Technitium DNS.

+
+
+
Drives: 10 Disks
+
Total Storage: 1.14 TiB Total
+
+
+ + +
+
+
+ Longhorn Aerodrome Storage + 140.44.4.11-27 +
+

Replicated low-latency distributed block storage backing Prometheus, Grafana, Authentik, AWX, and databases.

+
+
+
Drives: 7 Disks
+
Total Storage: 818 GiB Total
+
+
+ + +
+
+
+ + + Harvester HCI Distributed Storage (Spacedock Cluster) + + 140.44.1.0/24 (VLAN 3) +
+

Multi-node hypervisor storage fabric across 4 physical Harvester nodes backing VM virtual disks, container rootfs, and AI model inference storage.

+
+
+
Drives: 35 Disks (across 4 nodes)
+
Total Storage: 46.41 TiB Total (46.22 TiB avail)
+
+
+
+
+ + +
+
+
+ šŸ” +

Smart Home & Edge AI Fabric (VLAN 50)

+
+ 14.4.4.0/25 +
+ +
+
+
+ Home Assistant Yellow + 14.4.4.20:8123 +
+

Primary smart home automation hub, Zigbee/Z-Wave networks, Matter, local voice processing.

+
+ +
+
+ Tempest Weather Station + Direct HA Telemetry +
+

Local UDP & cloud backyard weather station (Temp, Feels Like, Humidity, Wind Speed rounded to 0.1Ā°).

+
+ +
+
+ NUT Master Daemon + 14.4.4.44:3493 +
+

Network UPS Tools master daemon monitoring central battery units, broadcast shutdown signals.

+
+ +
+
+ Hailo-8 Edge NPU Node + 140.44.4.7:8000 +
+

Dedicated Raspberry Pi 5 with 26 TOPS Hailo-8 AI acceleration running lightweight local LLMs.

+
+
+
+ +
+ +
+ + + + + + diff --git a/docs/snarfnet_homelab_topology.png b/docs/snarfnet_homelab_topology.png new file mode 100644 index 0000000..326d937 Binary files /dev/null and b/docs/snarfnet_homelab_topology.png differ diff --git a/homelab-gitops-repo b/homelab-gitops-repo index 7b2dd24..c7eb47f 160000 --- a/homelab-gitops-repo +++ b/homelab-gitops-repo @@ -1 +1 @@ -Subproject commit 7b2dd24526eab8f630631b41c28bf01acd0aec90 +Subproject commit c7eb47f6d8d9d030b82d6786da7c1ed3c2a954e5 diff --git a/scripts/add_kuma_bookmarks.py b/scripts/add_kuma_bookmarks.py new file mode 100644 index 0000000..4c082b3 --- /dev/null +++ b/scripts/add_kuma_bookmarks.py @@ -0,0 +1,58 @@ +import yaml + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + text = f.read() + +docs = list(yaml.safe_load_all(text)) +cm = docs[2] + +bookmarks_yaml = """- Quick Links: + - GitHub: + - abbr: GH + href: https://github.com/ + - UniFi Site Manager: + - abbr: UI + href: https://unifi.ui.com/ + - Cloudflare: + - abbr: CF + href: https://dash.cloudflare.com/ +- Grafana Telemetry: + - Systems Performance: + - abbr: SYS + href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861 + - Container Workloads: + - abbr: K8S + href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads + - AI Stack Intelligence: + - abbr: AI + href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence +- Uptime Status Pages: + - Unified Overview: + - abbr: ALL + href: https://uptime.snarfnet.net/status/default + - Core & VIP Fabric: + - abbr: VIP + href: https://uptime.snarfnet.net/status/core + - GitOps & CI/CD: + - abbr: OPS + href: https://uptime.snarfnet.net/status/gitops + - Edge & Ingress: + - abbr: EDG + href: https://uptime.snarfnet.net/status/edge +""" + +cm['data']['bookmarks.yaml'] = bookmarks_yaml + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f: + yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +print("Updated bookmarks in homepage.yaml successfully!") + diff --git a/scripts/add_topology_to_homepage.py b/scripts/add_topology_to_homepage.py new file mode 100644 index 0000000..f8184d2 --- /dev/null +++ b/scripts/add_topology_to_homepage.py @@ -0,0 +1,26 @@ +import yaml + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + text = f.read() + +docs = list(yaml.safe_load_all(text)) +cm = docs[2] + +services = yaml.safe_load(cm['data']['services.yaml']) +bookmarks = yaml.safe_load(cm['data']['bookmarks.yaml']) + +cm['data']['services.yaml'] = yaml.dump(services, sort_keys=False, default_flow_style=False, allow_unicode=True) +cm['data']['bookmarks.yaml'] = yaml.dump(bookmarks, sort_keys=False, default_flow_style=False, allow_unicode=True) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f: + yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +print("Cleaned up unicode strings in homepage.yaml!") diff --git a/scripts/add_zabbix_devices.py b/scripts/add_zabbix_devices.py new file mode 100644 index 0000000..3276d8d --- /dev/null +++ b/scripts/add_zabbix_devices.py @@ -0,0 +1,118 @@ +import json +import ssl +import urllib.request + +ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php" +ZABBIX_USER = "Admin" +ZABBIX_PASS = "zabbix" + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +def zbx_call(method, params, auth=None): + payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1} + headers = {"Content-Type": "application/json-rpc"} + if auth: + headers["Authorization"] = f"Bearer {auth}" + req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + resp = json.loads(r.read().decode("utf-8")) + if "error" in resp: + raise RuntimeError(f"Zabbix error in {method}: {resp['error']}") + return resp["result"] + +auth = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS}) + +# Check existing host names +existing_hosts = zbx_call("host.get", {"output": ["hostid", "host", "name"]}, auth) +existing_names = {h["host"].lower(): h["hostid"] for h in existing_hosts} + +# Create host groups if not exist +groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth) +group_map = {g["name"]: g["groupid"] for g in groups} + +for gname in ["Network Devices", "Environmental & Sensors", "Management & IPMI"]: + if gname not in group_map: + res = zbx_call("hostgroup.create", {"name": gname}, auth) + group_map[gname] = res["groupids"][0] + print(f"Created Host Group: {gname} (ID: {group_map[gname]})") + +devices_to_add = [ + { + "host": "idrac-spacedock4", + "name": "Dell PowerEdge R730 Node 4 iDRAC (idrac-spacedock4)", + "ip": "140.44.44.124", + "group": "Management & IPMI" + }, + { + "host": "usw-agg-core", + "name": "UniFi Switch Aggregation 10G Core", + "ip": "14.44.4.26", + "group": "Network Devices" + }, + { + "host": "usw-24-poe-closet", + "name": "UniFi Switch 24 PoE Closet", + "ip": "14.44.4.61", + "group": "Network Devices" + }, + { + "host": "usw-pro-24-poe-compute", + "name": "UniFi Switch Pro 24 PoE Compute", + "ip": "14.44.4.107", + "group": "Network Devices" + }, + { + "host": "usw-flex-attic", + "name": "UniFi Switch Flex Attic", + "ip": "14.44.4.98", + "group": "Network Devices" + }, + { + "host": "usw-flex-garage", + "name": "UniFi Switch Flex Garage", + "ip": "14.44.4.96", + "group": "Network Devices" + }, + { + "host": "tempest-weather-station", + "name": "Tempest Weather Station Gateway", + "ip": "14.4.4.47", + "group": "Environmental & Sensors" + }, + { + "host": "sense-energy-monitor", + "name": "Sense Home Energy Gateway", + "ip": "14.4.4.48", + "group": "Environmental & Sensors" + } +] + +for dev in devices_to_add: + hname = dev["host"] + if hname.lower() in existing_names: + print(f"[EXISTS] {hname}") + continue + params = { + "host": dev["host"], + "name": dev["name"], + "interfaces": [ + { + "type": 1, # Agent / ICMP + "main": 1, + "useip": 1, + "ip": dev["ip"], + "dns": "", + "port": "10050" + } + ], + "groups": [{"groupid": group_map[dev["group"]]}], + "templates": [{"templateid": "10564"}] # ICMP Ping + } + try: + res = zbx_call("host.create", params, auth) + print(f"[CREATED] {dev['name']} ({dev['ip']}) -> HostID {res['hostids'][0]}") + except Exception as e: + print(f"[FAILED] {dev['name']}: {e}") + diff --git a/scripts/apply_full_homepage.py b/scripts/apply_full_homepage.py new file mode 100644 index 0000000..ed6c12f --- /dev/null +++ b/scripts/apply_full_homepage.py @@ -0,0 +1,250 @@ +import yaml + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + text = f.read() + +docs = list(yaml.safe_load_all(text)) +cm = docs[2] + +services_yaml = """- Critical Services: + - Authentik: + icon: authentik.png + href: https://authentik.snarfnet.net + description: SSO & Identity Management + - PeaNUT UPS: + icon: peanut.png + href: https://portal.snarfnet.net + description: UPS Power & Battery Monitor + - Unifi Controller: + icon: unifi.png + href: https://unifi.ui.com + widget: + type: unifi + url: https://140.44.44.1 + username: Snarf + password: Katzir476!!! + +- GitOps & Automation: + - Gitea: + icon: gitea.png + href: https://git.snarfnet.net + description: Git Repositories + widget: + type: gitea + url: https://git.snarfnet.net + key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71 + - ArgoCD: + icon: argocd.png + href: https://argo.snarfnet.net + description: Declarative GitOps Reconciler + widget: + type: argocd + url: https://argo.snarfnet.net + key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0 + - AWX Automation: + icon: awx.png + href: https://awx.snarfnet.net + description: Ansible Automation Controller + - Semaphore: + icon: semaphore.png + href: https://semaphore.snarfnet.net + description: Playbook Execution Engine + +- Storage & Clusters: + - Longhorn Aerodrome: + icon: longhorn.png + href: https://longhorn.aerodrome.snarfnet.net + description: Aerodrome Distributed Flash Storage + - Longhorn Spaceport: + icon: longhorn.png + href: https://longhorn.spaceport.snarfnet.net + description: Spaceport Distributed Block Storage + - Rancher (Aerodrome): + icon: rancher.png + href: https://aerodrome.snarfnet.net + description: Kubernetes Management Console + widget: + type: customapi + url: https://aerodrome.snarfnet.net/v3/clusters/local + headers: + Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7" + mappings: + - field: capacity.cpu + label: CPU Cores + - field: capacity.pods + label: Max Pods + - field: state + label: Cluster State + - Harvester HCI (Spacedock): + icon: harvester.png + href: https://harvester.spacedock.snarfnet.net + description: HCI Virtualization Hypervisors + widget: + type: customapi + url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt + headers: + Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7" + mappings: + - field: capacity.cpu + label: CPU Cores + - field: capacity.pods + label: Max Pods + - field: state + label: HCI State + - Portainer Swarm: + icon: portainer.png + href: https://shipyard.snarfnet.net + description: Docker Swarm Container Fleet + widget: + type: portainer + url: https://140.44.4.71:9443 + env: 3 + key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8= + +- Infrastructure & Observability: + - NGINX Proxy Manager: + icon: nginx-proxy-manager + href: https://nginx.snarfnet.net + description: Reverse Proxy & SSL Certificates + - Uptime Kuma: + icon: uptime-kuma + href: https://uptime.snarfnet.net + description: Multi-Endpoint Availability Probes + widget: + type: uptimekuma + url: http://140.44.4.28:448 + slug: default + - Zabbix: + icon: zabbix.png + href: https://zabbix.snarfnet.net + description: Infrastructure Metrics & Alarms + - Grafana: + icon: grafana.png + href: https://grafana.snarfnet.net + description: Real-Time Metrics & AI Dashboards + widget: + type: grafana + url: https://grafana.snarfnet.net + username: admin + password: Katzir476!!! + - Core Node Glances: + icon: glances.png + href: http://140.44.4.11:61208 + description: Host Resource Monitor + +- Environment & Weather: + - Tempest Weather Station: + icon: weather.png + href: https://snarfhome.snarfnet.net + description: Backyard Weather Telemetry + widget: + type: homeassistant + url: http://14.4.4.20:8123 + key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4 + custom: + - template: "{{ states('sensor.st_00158555_temperature') | float(0) | round(1) }} Ā°F" + label: Outdoor Temp + - template: "{{ states('sensor.st_00158555_feels_like') | float(0) | round(1) }} Ā°F" + label: Feels Like + - template: "{{ states('sensor.st_00158555_humidity') | float(0) | round(1) }} %" + label: Humidity + - template: "{{ states('sensor.st_00158555_wind_speed') | float(0) | round(1) }} mph" + label: Wind Speed +""" + +settings_yaml = """title: SnarfNet Operations & Telemetry +headerStyle: boxed +color: slate +theme: dark +statusStyle: badge +background: + image: https://i.imgur.com/hOzzkCY.jpeg + blur: sm + opacity: 50 + brightness: 50 +cardBlur: sm +quicklaunch: + searchDescriptions: true + hideInternetSearch: true + hideVisitURL: true + hideVersion: true +layout: + Critical Services: + icon: mdi-shield-check + style: row + columns: 3 + GitOps & Automation: + icon: mdi-git + style: row + columns: 4 + Storage & Clusters: + icon: mdi-server-network + style: row + columns: 3 + Infrastructure & Observability: + icon: mdi-chart-timeline-variant + style: row + columns: 3 + Environment & Weather: + icon: mdi-weather-partly-cloudy + style: row + columns: 1 +showStats: true +providers: + longhorn: + url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500 +""" + +widgets_yaml = """- kubernetes: + cluster: + show: true + cpu: true + memory: true + show_names: true + nodes: + show: false +- longhorn: + total: true + nodes: false +- resources: + label: System + backend: resources + cpu: true + expanded: true + memory: true + disk: / +- search: + provider: duckduckgo + target: _blank +""" + +bookmarks_yaml = """- Quick Links: + - GitHub: + - abbr: GH + href: https://github.com/ + - UniFi Site Manager: + - abbr: UI + href: https://unifi.ui.com/ + - Cloudflare: + - abbr: CF + href: https://dash.cloudflare.com/ +""" + +cm['data']['services.yaml'] = services_yaml +cm['data']['settings.yaml'] = settings_yaml +cm['data']['widgets.yaml'] = widgets_yaml +cm['data']['bookmarks.yaml'] = bookmarks_yaml +cm['data']['kubernetes.yaml'] = "mode: cluster\n" + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f: + yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +print("Saved clean homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml successfully!") diff --git a/scripts/apply_grafana_dashboards.py b/scripts/apply_grafana_dashboards.py new file mode 100644 index 0000000..581237c --- /dev/null +++ b/scripts/apply_grafana_dashboards.py @@ -0,0 +1,33 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r", encoding="utf-8") as f: + grafana_yaml = f.read() + +stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_updated.yaml") +stdin.write(grafana_yaml) +stdin.close() +stdout.channel.recv_exit_status() + +print("Applying Grafana manifests:") +print(run("kubectl apply -f /tmp/grafana_updated.yaml")) + +print("\nRestarting Grafana:") +print(run("kubectl rollout restart deployment/grafana -n default")) +print(run("kubectl rollout status deployment/grafana -n default --timeout=60s")) + +print("\nWaiting for Grafana to finish initialization...") +time.sleep(10) + +c.close() + diff --git a/scripts/apply_grafana_server_side.py b/scripts/apply_grafana_server_side.py new file mode 100644 index 0000000..800c39c --- /dev/null +++ b/scripts/apply_grafana_server_side.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +print("Applying server-side:") +print(run("kubectl apply -f /tmp/grafana_full.yaml --server-side --force-conflicts")) +print("\nRollout restart:") +print(run("kubectl rollout restart deployment/grafana -n default")) +print(run("kubectl rollout status deployment/grafana -n default --timeout=60s")) + +c.close() + diff --git a/scripts/apply_homepage.py b/scripts/apply_homepage.py new file mode 100644 index 0000000..4378880 --- /dev/null +++ b/scripts/apply_homepage.py @@ -0,0 +1,31 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + return out + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + content = f.read() + +stdin, stdout, stderr = c.exec_command('cat > /tmp/homepage.yaml') +stdin.write(content) +stdin.close() +stdout.read() + +run("ls -lh /tmp/homepage.yaml") +run("kubectl apply -f /tmp/homepage.yaml") +run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure") +run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s") + +c.close() diff --git a/scripts/apply_prom_grafana.py b/scripts/apply_prom_grafana.py new file mode 100644 index 0000000..a5b755a --- /dev/null +++ b/scripts/apply_prom_grafana.py @@ -0,0 +1,45 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r") as f: + prom_yaml = f.read() + +stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml") +stdin.write(prom_yaml) +stdin.close() +stdout.channel.recv_exit_status() + +print("Applying Prometheus:") +print(run("kubectl apply -f /tmp/prometheus.yaml")) + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r") as f: + grafana_yaml = f.read() + +stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_full.yaml") +stdin.write(grafana_yaml) +stdin.close() +stdout.channel.recv_exit_status() + +print("\nApplying Grafana configuration & dashboards:") +print(run("kubectl apply -f /tmp/grafana_full.yaml")) + +print("\nRestarting Grafana:") +print(run("kubectl rollout restart deployment/grafana -n default")) +print(run("kubectl rollout status deployment/grafana -n default --timeout=60s")) + +print("\nChecking Prometheus pod:") +time.sleep(5) +print(run("kubectl get pods -l app.kubernetes.io/name=prometheus")) + +c.close() + diff --git a/scripts/apply_prom_update.py b/scripts/apply_prom_update.py new file mode 100644 index 0000000..6ccc438 --- /dev/null +++ b/scripts/apply_prom_update.py @@ -0,0 +1,36 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f: + prom_yaml = f.read() + +stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_updated.yaml") +stdin.write(prom_yaml) +stdin.close() +stdout.channel.recv_exit_status() + +print("Applying Prometheus:") +print(run("kubectl apply -f /tmp/prometheus_updated.yaml")) + +print("\nRestarting Prometheus:") +print(run("kubectl rollout restart deployment/prometheus -n default")) +print(run("kubectl rollout status deployment/prometheus -n default --timeout=60s")) + +print("\nWaiting 15 seconds for scrape loop...") +time.sleep(15) + +print("\nPrometheus active targets count:") +print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets | grep -o '\"health\":\"up\"' | wc -l")) + +c.close() + diff --git a/scripts/capture_screenshot.py b/scripts/capture_screenshot.py new file mode 100644 index 0000000..56bd062 --- /dev/null +++ b/scripts/capture_screenshot.py @@ -0,0 +1,27 @@ +import subprocess +import os + +edge_path = r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe" +out_png = os.path.abspath("docs/snarfnet_homelab_topology.png") +html_url = "file:///" + os.path.abspath("docs/snarfnet_homelab_topology.html").replace("\\", "/") + +cmd = [ + edge_path, + "--headless", + "--disable-gpu", + "--no-sandbox", + "--hide-scrollbars", + "--window-size=1920,2400", + f"--screenshot={out_png}", + html_url +] + +print("Running cmd:", " ".join(cmd)) +res = subprocess.run(cmd, capture_output=True, text=True, timeout=30) +print("Return code:", res.returncode) +print("Stdout:", res.stdout) +print("Stderr:", res.stderr) +print("File exists?", os.path.exists(out_png)) +if os.path.exists(out_png): + print("File size:", os.path.getsize(out_png)) + diff --git a/scripts/cat_prom_conf.py b/scripts/cat_prom_conf.py new file mode 100644 index 0000000..751b32a --- /dev/null +++ b/scripts/cat_prom_conf.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/prometheus -- cat /etc/prometheus/prometheus.yml")) +c.close() + diff --git a/scripts/check_aero_metrics_sources.py b/scripts/check_aero_metrics_sources.py new file mode 100644 index 0000000..63cb2ce --- /dev/null +++ b/scripts/check_aero_metrics_sources.py @@ -0,0 +1,24 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Nodes on Aerodrome:") +print(run("kubectl get nodes -o wide")) + +print("\nDaemonSets on Aerodrome:") +print(run("kubectl get ds -A")) + +print("\nServices matching metrics:") +print(run("kubectl get svc -A | grep -E 'metric|node|cadvisor|exporter'")) + +print("\nKubelet metrics on node 140.44.4.11 (port 10250 or 10255):") +print(run("curl -sk https://localhost:10250/metrics/cadvisor | head -n 10")) + +c.close() + diff --git a/scripts/check_argo_secret.py b/scripts/check_argo_secret.py new file mode 100644 index 0000000..2aa2e15 --- /dev/null +++ b/scripts/check_argo_secret.py @@ -0,0 +1,16 @@ +import paramiko +import base64 + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Argo initial secret:") +print(run("kubectl get secret argocd-initial-admin-secret -n infrastructure -o jsonpath='{.data.password}'")) + +c.close() + diff --git a/scripts/check_argocd_apps.py b/scripts/check_argocd_apps.py new file mode 100644 index 0000000..dfb4386 --- /dev/null +++ b/scripts/check_argocd_apps.py @@ -0,0 +1,73 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0" +headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" +} + +try: + resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10) + data = resp.json() + items = data.get("items", []) + print(f"Total Applications: {len(items)}\n") + + out_of_sync = [] + synced = [] + + for app in items: + name = app.get("metadata", {}).get("name") + sync_status = app.get("status", {}).get("sync", {}).get("status") + health_status = app.get("status", {}).get("health", {}).get("status") + dest_server = app.get("spec", {}).get("destination", {}).get("server") + dest_ns = app.get("spec", {}).get("destination", {}).get("namespace") + repo = app.get("spec", {}).get("source", {}).get("repoURL") + path = app.get("spec", {}).get("source", {}).get("path") + + # Check operation or sync conditions/messages + operation_state = app.get("status", {}).get("operationState", {}) + op_phase = operation_state.get("phase") + op_msg = operation_state.get("message") + conditions = app.get("status", {}).get("conditions", []) + + info = { + "name": name, + "sync": sync_status, + "health": health_status, + "dest_ns": dest_ns, + "dest_server": dest_server, + "repo": repo, + "path": path, + "op_phase": op_phase, + "op_msg": op_msg, + "conditions": conditions + } + + if sync_status != "Synced" or health_status not in ["Healthy", None]: + out_of_sync.append(info) + else: + synced.append(info) + + print(f"=== OUT OF SYNC / DEGRADED APPS ({len(out_of_sync)}) ===") + for a in out_of_sync: + print(f"\nApp: {a['name']}") + print(f" Sync: {a['sync']} | Health: {a['health']}") + print(f" Dest: {a['dest_ns']} on {a['dest_server']}") + print(f" Path: {a['path']}") + if a['op_phase']: + print(f" Op Phase: {a['op_phase']}") + if a['op_msg']: + print(f" Op Msg: {a['op_msg']}") + if a['conditions']: + print(f" Conditions: {a['conditions']}") + + print(f"\n=== SYNCED APPS ({len(synced)}) ===") + for a in synced: + print(f" {a['name']}: Sync={a['sync']}, Health={a['health']}, NS={a['dest_ns']}") + +except Exception as e: + print("Error querying Argo CD API:", e) + diff --git a/scripts/check_argocd_resources.py b/scripts/check_argocd_resources.py new file mode 100644 index 0000000..8195023 --- /dev/null +++ b/scripts/check_argocd_resources.py @@ -0,0 +1,29 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0" +headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" +} + +resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10) +data = resp.json() +items = data.get("items", []) + +for app in items: + name = app.get("metadata", {}).get("name") + sync = app.get("status", {}).get("sync", {}).get("status") + health = app.get("status", {}).get("health", {}).get("status") + resources = app.get("status", {}).get("resources", []) + + out_resources = [r for r in resources if r.get("status") != "Synced" or r.get("health", {}).get("status") not in ["Healthy", None]] + + print(f"[{name}] Sync: {sync}, Health: {health}, Total Resources: {len(resources)}") + if out_resources: + print(f" -> Out-of-sync / unhealthy resources:") + for r in out_resources: + print(f" {r.get('kind')}/{r.get('name')}: Sync={r.get('status')}, Health={r.get('health', {}).get('status')}") + diff --git a/scripts/check_authentik_proxy_providers.py b/scripts/check_authentik_proxy_providers.py new file mode 100644 index 0000000..fc33dcb --- /dev/null +++ b/scripts/check_authentik_proxy_providers.py @@ -0,0 +1,24 @@ +import urllib.request, ssl, json + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +AK_BASE = "https://authentik.snarfnet.net" +AK_TOKEN = "SnarfNetPermanentAutomationToken2026" + +def req(endpoint): + url = f"{AK_BASE}/api/v3/{endpoint}" + r = urllib.request.Request(url, headers={"Authorization": f"Bearer {AK_TOKEN}"}) + with urllib.request.urlopen(r, timeout=10, context=ctx) as res: + return json.loads(res.read().decode()) + +print("=== Authentik Providers (Proxy) ===") +providers = req("providers/proxy/") +for p in providers.get("results", []): + print(p["pk"], p["name"], p.get("internal_host"), p.get("external_host"), p.get("mode")) + +print("\n=== Authentik Outposts ===") +outposts = req("outposts/instances/") +for o in outposts.get("results", []): + print(o["pk"], o["name"], o["type"], [p for p in o.get("providers", [])]) diff --git a/scripts/check_awx_monitor.py b/scripts/check_awx_monitor.py new file mode 100644 index 0000000..0b679c2 --- /dev/null +++ b/scripts/check_awx_monitor.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, url, port, accepted_statuscodes_json FROM monitor WHERE name LIKE '%AWX%';" """)) +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM heartbeat WHERE monitor_id IN (SELECT id FROM monitor WHERE name LIKE '%AWX%') ORDER BY id DESC LIMIT 5;" """)) + +c.close() + diff --git a/scripts/check_cm_json.py b/scripts/check_cm_json.py new file mode 100644 index 0000000..2f2d072 --- /dev/null +++ b/scripts/check_cm_json.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('kubectl get cm homepage-2-1744068158 -n infrastructure -o json') +import json +cm = json.loads(o.read().decode()) +print("Data keys in ConfigMap:", list(cm.get('data', {}).keys())) +print("\n--- settings.yaml ---") +print(cm.get('data', {}).get('settings.yaml')) +print("\n--- widgets.yaml ---") +print(cm.get('data', {}).get('widgets.yaml')) + +c.close() + diff --git a/scripts/check_dcgm_status.py b/scripts/check_dcgm_status.py new file mode 100644 index 0000000..2ab6f36 --- /dev/null +++ b/scripts/check_dcgm_status.py @@ -0,0 +1,21 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'") + out = stdout.read().decode().strip() + return out + +time.sleep(5) +print("Pods status:") +print(run("kubectl get pods -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o wide")) +print("Logs from one pod:") +pod = run("kubectl get pod -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl logs -n ai-stack {pod} --tail=20")) + +c.close() + diff --git a/scripts/check_deleted_at.py b/scripts/check_deleted_at.py new file mode 100644 index 0000000..b7ca7c0 --- /dev/null +++ b/scripts/check_deleted_at.py @@ -0,0 +1,26 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("""python3 -c " +import sqlite3 +con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite') +cur = con.cursor() +for r in cur.execute('SELECT id, user_id, title, pinned, \"order\", deleted_at FROM items LIMIT 10;').fetchall(): + print(r) +" """) + +c.close() + diff --git a/scripts/check_deployment_spec.py b/scripts/check_deployment_spec.py new file mode 100644 index 0000000..242d0d5 --- /dev/null +++ b/scripts/check_deployment_spec.py @@ -0,0 +1,14 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl get deployment grafana -o jsonpath='{.spec.template.spec.containers[0].volumeMounts}'")) + +c.close() + diff --git a/scripts/check_direct_apis.py b/scripts/check_direct_apis.py new file mode 100644 index 0000000..b57cb33 --- /dev/null +++ b/scripts/check_direct_apis.py @@ -0,0 +1,24 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def test(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res[:300]) + if err: + print("[stderr]", err[:200]) + +test("curl -m 3 -s -I http://140.44.4.28:448/ || echo 'Timeout 140.44.4.28:448'") +test("curl -m 3 -s -I http://140.44.4.31:448/ || echo 'Timeout 140.44.4.31:448'") +test("curl -m 3 -s http://140.44.4.70:8050/api/v1/devices || echo 'Failed PeaNUT'") +test("curl -m 3 -s -H 'Authorization: token 47c10c54574131c007302b246352f653facb5062' http://140.44.4.62:90/api/v1/user || echo 'Failed Gitea'") + +c.close() diff --git a/scripts/check_down_monitors.py b/scripts/check_down_monitors.py new file mode 100644 index 0000000..c622dbb --- /dev/null +++ b/scripts/check_down_monitors.py @@ -0,0 +1,41 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +try: + r = requests.get("http://140.44.4.28:448/api/status-page/heartbeat/default", verify=False, timeout=10) + data = r.json() + heartbeats = data.get("heartbeatList", {}) + down = [] + up = [] + + # Also fetch status page monitor list to map IDs to names + r_sp = requests.get("http://140.44.4.28:448/api/status-page/default", verify=False, timeout=10) + sp_data = r_sp.json() + + id_to_name = {} + for group in sp_data.get("publicGroupList", []): + for m in group.get("monitorList", []): + id_to_name[str(m.get("id"))] = m.get("name") + + for mid, beats in heartbeats.items(): + name = id_to_name.get(str(mid), f"Monitor #{mid}") + if beats: + last = beats[-1] + if last.get("status") == 1: + up.append((name, last.get("ping"))) + else: + down.append((name, last.get("msg"))) + + print(f"Total Monitors: {len(heartbeats)}") + print(f"Up: {len(up)} | Down: {len(down)}") + if down: + print("\nDown Monitors:") + for name, msg in down: + print(f" āŒ {name}: {msg}") + else: + print("\nAll monitors are UP! šŸŽ‰") +except Exception as e: + print("Error:", e) + diff --git a/scripts/check_gaming_pc.py b/scripts/check_gaming_pc.py new file mode 100644 index 0000000..93590c0 --- /dev/null +++ b/scripts/check_gaming_pc.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Ping 140.44.4.80:") +print(run("ping -c 2 -W 1 140.44.4.80 || true")) +print("ARP table for bc:24:11:67:16:cd:") +print(run("ip neigh | grep -i 'bc:24:11:67:16:cd' || arp -a | grep -i 'bc:24:11:67:16:cd' || true")) + +c.close() + diff --git a/scripts/check_glances_host.py b/scripts/check_glances_host.py new file mode 100644 index 0000000..bdce4d3 --- /dev/null +++ b/scripts/check_glances_host.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +print("ps aux | grep glances:") +print(run("ps aux | grep glances")) + +print("\nnetstat / ss listening:") +print(run("ss -tulpn | grep 61208 || ss -tulpn | grep glances")) + +c.close() + diff --git a/scripts/check_gotify_files.py b/scripts/check_gotify_files.py new file mode 100644 index 0000000..a940d96 --- /dev/null +++ b/scripts/check_gotify_files.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("ls -la /mnt/swarm_shared/legacy_volumes/gotify/_data/")) + +c.close() diff --git a/scripts/check_gotify_idx.py b/scripts/check_gotify_idx.py new file mode 100644 index 0000000..d2bce9b --- /dev/null +++ b/scripts/check_gotify_idx.py @@ -0,0 +1,22 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """import sqlite3 +conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/gotify.db') +cur = conn.cursor() +indexes = cur.execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name='applications'").fetchall() +for idx in indexes: + print(idx) +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/check_idx.py && python3 /tmp/check_idx.py") +stdin.write(py_script) +stdin.close() +print(stdout.read().decode()) +print(stderr.read().decode()) + +c.close() + diff --git a/scripts/check_gotify_mount.py b/scripts/check_gotify_mount.py new file mode 100644 index 0000000..262180c --- /dev/null +++ b/scripts/check_gotify_mount.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +print("Gotify inspect mounts:") +print(run("docker service inspect gotify_gotify --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")) + +c.close() + diff --git a/scripts/check_gotify_service.py b/scripts/check_gotify_service.py new file mode 100644 index 0000000..d1df80e --- /dev/null +++ b/scripts/check_gotify_service.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +print("Gotify service ps:") +print(run("docker service ps gotify_gotify --no-trunc | head -n 10")) + +print("\nGotify service logs:") +print(run("docker service logs gotify_gotify --tail=25")) + +c.close() + diff --git a/scripts/check_grafana_api.py b/scripts/check_grafana_api.py new file mode 100644 index 0000000..06132d1 --- /dev/null +++ b/scripts/check_grafana_api.py @@ -0,0 +1,27 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +auth = ('admin', 'd0KJLZJXUrbIhW1sjU8lrdFYj7VXts26V7Jw7xQr') + +try: + r = requests.get('https://grafana.snarfnet.net/api/datasources', auth=auth, verify=False, timeout=10) + print("Datasources Status:", r.status_code) + if r.status_code == 200: + for ds in r.json(): + print(f" - ID: {ds.get('id')}, Name: {ds.get('name')}, Type: {ds.get('type')}, URL: {ds.get('url')}, isDefault: {ds.get('isDefault')}") +except Exception as e: + print("Error getting datasources:", e) + +try: + r = requests.get('https://grafana.snarfnet.net/api/search', auth=auth, verify=False, timeout=10) + print("Dashboards Status:", r.status_code) + if r.status_code == 200: + dashboards = r.json() + print(f"Total dashboards: {len(dashboards)}") + for d in dashboards[:15]: + print(f" - {d.get('title')} (uri: {d.get('uri')}, type: {d.get('type')})") +except Exception as e: + print("Error getting dashboards:", e) + diff --git a/scripts/check_grafana_dashboards_live.py b/scripts/check_grafana_dashboards_live.py new file mode 100644 index 0000000..2561a4c --- /dev/null +++ b/scripts/check_grafana_dashboards_live.py @@ -0,0 +1,15 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +auth = ("admin", "Katzir476!!!") +url = "https://grafana.snarfnet.net/api/search" + +resp = requests.get(url, auth=auth, verify=False, timeout=10) +print("Status:", resp.status_code) +dashboards = resp.json() +print(f"Total Dashboards Found: {len(dashboards)}") +for d in dashboards: + print(f" - [{d.get('type')}] Title: {d.get('title')} | UID: {d.get('uid')} | URL: {d.get('url')}") + diff --git a/scripts/check_grafana_diff.py b/scripts/check_grafana_diff.py new file mode 100644 index 0000000..58e43f7 --- /dev/null +++ b/scripts/check_grafana_diff.py @@ -0,0 +1,14 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl diff -f /tmp/grafana_full.yaml || true")) + +c.close() + diff --git a/scripts/check_grafana_info.py b/scripts/check_grafana_info.py new file mode 100644 index 0000000..00daaf8 --- /dev/null +++ b/scripts/check_grafana_info.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Namespaces on Aerodrome:") +print(run("kubectl get ns")) + +print("Grafana pod:") +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print(pod) +print("Grafana admin secret:") +print(run("kubectl get secret grafana -o jsonpath='{.data.admin-password}' | base64 -d")) + +c.close() + diff --git a/scripts/check_grafana_internal.py b/scripts/check_grafana_internal.py new file mode 100644 index 0000000..f0c9f77 --- /dev/null +++ b/scripts/check_grafana_internal.py @@ -0,0 +1,23 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print("Pod:", pod) +print("ConfigMap grafana:") +print(run("kubectl get cm grafana -o yaml")) + +print("Testing local curl inside pod to localhost:3000/api/health:") +print(run(f"kubectl exec {pod} -- wget -qO- http://localhost:3000/api/health")) + +print("Grafana env:") +print(run(f"kubectl exec {pod} -- env | grep -i grafana")) + +c.close() + diff --git a/scripts/check_grafana_load.py b/scripts/check_grafana_load.py new file mode 100644 index 0000000..34275cc --- /dev/null +++ b/scripts/check_grafana_load.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print("Grafana pod:", pod) +print("Grafana provisioning logs:") +print(run(f"kubectl logs {pod} | grep -i -E 'provision|datasource|dashboard' | tail -n 25")) + +c.close() + diff --git a/scripts/check_grafana_mounts.py b/scripts/check_grafana_mounts.py new file mode 100644 index 0000000..5159306 --- /dev/null +++ b/scripts/check_grafana_mounts.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl exec {pod} -- ls -la /etc/grafana/provisioning/datasources")) +print(run(f"kubectl exec {pod} -- cat /etc/grafana/provisioning/datasources/datasources.yaml")) +print(run(f"kubectl exec {pod} -- ls -la /etc/grafana/provisioning/dashboards")) +print(run(f"kubectl exec {pod} -- cat /etc/grafana/provisioning/dashboards/dashboards.yaml")) +print(run(f"kubectl exec {pod} -- ls -la /var/lib/grafana-dashboards")) + +c.close() + diff --git a/scripts/check_grafana_rollout.py b/scripts/check_grafana_rollout.py new file mode 100644 index 0000000..5691662 --- /dev/null +++ b/scripts/check_grafana_rollout.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl get pods -l app.kubernetes.io/name=grafana")) +c.close() + diff --git a/scripts/check_grafana_secret.py b/scripts/check_grafana_secret.py new file mode 100644 index 0000000..35d699f --- /dev/null +++ b/scripts/check_grafana_secret.py @@ -0,0 +1,15 @@ +import paramiko +import base64 +import json + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('kubectl get secret grafana -n default -o json') +data = json.loads(o.read().decode()).get('data', {}) +for k, v in data.items(): + print(k, ":", base64.b64decode(v).decode('utf-8', errors='ignore')) + +c.close() + diff --git a/scripts/check_grafana_sqlite.py b/scripts/check_grafana_sqlite.py new file mode 100644 index 0000000..c19a07d --- /dev/null +++ b/scripts/check_grafana_sqlite.py @@ -0,0 +1,30 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """import sqlite3 +conn = sqlite3.connect('/tmp/grafana_real.db') +cur = conn.cursor() +tables = [t[0] for t in cur.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall()] +for t in tables: + if 'source' in t or 'dash' in t or 'user' in t: + print('Matching table:', t) + +print('--- data_source content ---') +for row in cur.execute('SELECT * FROM data_source').fetchall(): + print(row) + +print('--- dashboard content ---') +for row in cur.execute('SELECT id, title, slug, uid FROM dashboard').fetchall(): + print(row) +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/inspect_grafana.py && python3 /tmp/inspect_grafana.py") +stdin.write(py_script) +stdin.close() +print(stdout.read().decode()) +print(stderr.read().decode()) + +c.close() diff --git a/scripts/check_grafana_success.py b/scripts/check_grafana_success.py new file mode 100644 index 0000000..63c25ee --- /dev/null +++ b/scripts/check_grafana_success.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print("Grafana pod:", pod) +print("Provisioning logs:") +print(run(f"kubectl logs {pod} | grep -i -E 'provision|datasource|dashboard'")) + +c.close() + diff --git a/scripts/check_ha_proxy_js.py b/scripts/check_ha_proxy_js.py new file mode 100644 index 0000000..0499797 --- /dev/null +++ b/scripts/check_ha_proxy_js.py @@ -0,0 +1,31 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +py_cmd = """ +node -e ' +const fs = require("fs"); +const content = fs.readFileSync("/app/.next/server/pages/api/services/proxy.js", "utf8"); +const idx = content.indexOf("states/"); +if (idx !== -1) { + console.log("Found states/ at", idx); + console.log(content.slice(Math.max(0, idx - 500), idx + 1000)); +} else { + const idx2 = content.indexOf("homeassistant"); + console.log("Found homeassistant at", idx2); + console.log(content.slice(Math.max(0, idx2 - 500), idx2 + 1000)); +} +' +""" +stdin, stdout, stderr = c.exec_command(f'kubectl exec -n infrastructure {pod} -- sh -c \'{py_cmd}\'') +print(stdout.read().decode()) +print(stderr.read().decode()) + +c.close() diff --git a/scripts/check_harvester_capacity.py b/scripts/check_harvester_capacity.py new file mode 100644 index 0000000..a9a3aa2 --- /dev/null +++ b/scripts/check_harvester_capacity.py @@ -0,0 +1,25 @@ +import paramiko, json + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; kubectl get nodes -o json'" +stdin, stdout, stderr = c.exec_command(cmd) +out = stdout.read().decode() +data = json.loads(out) + +total_cpu = 0 +total_mem_kb = 0 +for n in data['items']: + cpu = int(n['status']['capacity']['cpu']) + mem_str = n['status']['capacity']['memory'] + if mem_str.endswith('Ki'): + mem_kb = int(mem_str[:-2]) + total_cpu += cpu + total_mem_kb += mem_kb + +ram_gib = total_mem_kb / (1024 * 1024) +print(f"Spacedock Harvester: nodes = {len(data['items'])}, cpu = {total_cpu}, ram = {ram_gib:.1f} GiB") +c.close() + diff --git a/scripts/check_harvester_prom.py b/scripts/check_harvester_prom.py new file mode 100644 index 0000000..05f7070 --- /dev/null +++ b/scripts/check_harvester_prom.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'") + out = stdout.read().decode().strip() + return out + +print("=== Check all in cattle-monitoring-system on Harvester ===") +print(run("kubectl get all -n cattle-monitoring-system")) + +print("\n=== Check CRDs in monitoring ===") +print(run("kubectl get crd | grep -i monitoring")) + +print("\n=== Check Prometheus resources ===") +print(run("kubectl get prometheus -A")) + +c.close() diff --git a/scripts/check_harvester_storage.py b/scripts/check_harvester_storage.py new file mode 100644 index 0000000..c52cb17 --- /dev/null +++ b/scripts/check_harvester_storage.py @@ -0,0 +1,33 @@ +import paramiko, json + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + full_cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; " + cmd + "'" + stdin, stdout, stderr = c.exec_command(full_cmd) + return stdout.read().decode().strip() + +print("=== Blockdevices on Harvester ===") +print(run("kubectl get blockdevices.harvesterhci.io -A -o wide")) + +print("\n=== Longhorn storage on Harvester nodes ===") +out = run("kubectl get nodes.longhorn.io -n longhorn-system -o json") +data = json.loads(out) +total_max = 0 +total_avail = 0 +num_disks = 0 +for n in data['items']: + disks = n.get('status', {}).get('diskStatus', {}) + for d_name, d_val in disks.items(): + num_disks += 1 + total_max += d_val.get('storageMaximum', 0) + total_avail += d_val.get('storageAvailable', 0) + +tib = total_max / (1024**4) +avail_tib = total_avail / (1024**4) +print(f"Harvester Disks: {num_disks}, Total Storage: {tib:.2f} TiB, Available: {avail_tib:.2f} TiB") + +c.close() + diff --git a/scripts/check_heartbeats.py b/scripts/check_heartbeats.py new file mode 100644 index 0000000..96bfcf4 --- /dev/null +++ b/scripts/check_heartbeats.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print(f"Pod: {pod}") +print("Recent heartbeats summary:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e 'USE kuma; SELECT status, count(*) FROM heartbeat GROUP BY status;'")) + +print("\nSample recent heartbeats:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e 'USE kuma; SELECT m.name, h.status, h.msg, h.ping FROM heartbeat h JOIN monitor m ON h.monitor_id = m.id ORDER BY h.time DESC LIMIT 15;'")) + +c.close() + diff --git a/scripts/check_heimdall_active.py b/scripts/check_heimdall_active.py new file mode 100644 index 0000000..bf92d95 --- /dev/null +++ b/scripts/check_heimdall_active.py @@ -0,0 +1,31 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("""python3 -c " +import sqlite3 +con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite') +cur = con.cursor() +print('Active items (deleted_at IS NULL):', cur.execute('SELECT count(*) FROM items WHERE deleted_at IS NULL;').fetchone()[0]) +print('Deleted items (deleted_at IS NOT NULL):', cur.execute('SELECT count(*) FROM items WHERE deleted_at IS NOT NULL;').fetchone()[0]) +print('Active items duplicates:') +print(cur.execute('SELECT count(*), title FROM items WHERE deleted_at IS NULL GROUP BY title HAVING count(*) > 1;').fetchall()) +print('Active pinned items:') +for r in cur.execute('SELECT id, title, \"order\", pinned FROM items WHERE deleted_at IS NULL AND pinned = 1 ORDER BY \"order\";').fetchall(): + print(r) +" """) + +c.close() + diff --git a/scripts/check_heimdall_integrity.py b/scripts/check_heimdall_integrity.py new file mode 100644 index 0000000..e70d767 --- /dev/null +++ b/scripts/check_heimdall_integrity.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'PRAGMA integrity_check;'") +run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1;'") +run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'SELECT id, title, pinned, \"order\" FROM items ORDER BY \"order\", title LIMIT 30;'") + +c.close() + diff --git a/scripts/check_heimdall_pinned.py b/scripts/check_heimdall_pinned.py new file mode 100644 index 0000000..e1d05a3 --- /dev/null +++ b/scripts/check_heimdall_pinned.py @@ -0,0 +1,27 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("""python3 -c " +import sqlite3 +con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite') +cur = con.cursor() +print('Active items breakdown:') +for r in cur.execute('SELECT id, user_id, title, url, pinned, \"order\", deleted_at FROM items WHERE deleted_at IS NULL LIMIT 20;').fetchall(): + print(r) +" """) + +c.close() + diff --git a/scripts/check_heimdall_python.py b/scripts/check_heimdall_python.py new file mode 100644 index 0000000..f9d0d8c --- /dev/null +++ b/scripts/check_heimdall_python.py @@ -0,0 +1,24 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +run("which sqlite3 || echo 'sqlite3 not installed on host'") +run("python3 -c \"import sqlite3; con=sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite'); cur=con.cursor(); print(cur.execute('PRAGMA integrity_check;').fetchall()); print('Total items:', cur.execute('SELECT count(*) FROM items;').fetchone()[0]); print('Duplicates:', cur.execute('SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1;').fetchall())\"") + +c.close() + diff --git a/scripts/check_homepage_api.py b/scripts/check_homepage_api.py new file mode 100644 index 0000000..d53f66b --- /dev/null +++ b/scripts/check_homepage_api.py @@ -0,0 +1,26 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +s = requests.Session() +r = s.get('https://homepage.snarfnet.net/api/services', verify=False) +print("Status /api/services:", r.status_code) +try: + data = r.json() + print("Services JSON keys/length:", len(data)) + for group in data: + for title, items in group.items(): + print(f"Group: {title}") + for item in items: + print(f" - {item.get('name')}: widget={item.get('widget')}") +except Exception as e: + print("Error:", e, r.text[:200]) + +r2 = s.get('https://homepage.snarfnet.net/api/widgets', verify=False) +print("Status /api/widgets:", r2.status_code) +try: + print("Widgets:", json.dumps(r2.json(), indent=2)) +except Exception as e: + print("Error widgets:", e, r2.text[:200]) + diff --git a/scripts/check_homepage_info.py b/scripts/check_homepage_info.py new file mode 100644 index 0000000..0a78f3a --- /dev/null +++ b/scripts/check_homepage_info.py @@ -0,0 +1,11 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +_, o, _ = c.exec_command('kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath="{.spec.template.spec.containers[0].image}"') +print("Image:", o.read().decode()) +_, o, _ = c.exec_command('kubectl logs deployment/homepage-2-1744068158 -n infrastructure --tail=20') +print("Logs:\n", o.read().decode()) +c.close() + diff --git a/scripts/check_homepage_pod_logs.py b/scripts/check_homepage_pod_logs.py new file mode 100644 index 0000000..ee01dd8 --- /dev/null +++ b/scripts/check_homepage_pod_logs.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print("Logs from Homepage pod:") +print(run(f"kubectl logs -n infrastructure {pod} --tail=30")) + +c.close() + diff --git a/scripts/check_homepage_rbac.py b/scripts/check_homepage_rbac.py new file mode 100644 index 0000000..62001f2 --- /dev/null +++ b/scripts/check_homepage_rbac.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print("Pod:", pod) + +print("Check can-i list nodes:") +print(run(f"kubectl auth can-i list nodes --as=system:serviceaccount:infrastructure:default")) +print("Check can-i get nodes.metrics.k8s.io:") +print(run(f"kubectl auth can-i get nodes.metrics.k8s.io --as=system:serviceaccount:infrastructure:default")) + +c.close() + diff --git a/scripts/check_homepage_routes.py b/scripts/check_homepage_routes.py new file mode 100644 index 0000000..bdbaf4a --- /dev/null +++ b/scripts/check_homepage_routes.py @@ -0,0 +1,15 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +s = requests.Session() +r = s.get('https://homepage.snarfnet.net', verify=False) +print("Page length:", len(r.text)) +import re +apis = re.findall(r'/api/[a-zA-Z0-9_/]+', r.text) +print("Found API routes:", set(apis)) + +# Also test hitting the Home Assistant custom widget api if any +# In Homepage, the widget endpoint is /api/widgets/homeassistant or similar or /api/services/... +# Let's check pod files to see what routes exist + diff --git a/scripts/check_icon_resolver.py b/scripts/check_icon_resolver.py new file mode 100644 index 0000000..36907b4 --- /dev/null +++ b/scripts/check_icon_resolver.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") + +# Search for github raw or cdn in server.js or .next +print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'cdn.jsdelivr.net' /app/.next/server/ | head -n 5")) +print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'dashboard-icons' /app/src/")) + +c.close() + diff --git a/scripts/check_kuma_db_backups.py b/scripts/check_kuma_db_backups.py new file mode 100644 index 0000000..d52ab00 --- /dev/null +++ b/scripts/check_kuma_db_backups.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print("Pod:", pod) +print("Check for any .sql or .db files in /app/data:") +print(run(f"kubectl exec -n monitoring {pod} -- find /app/data -name '*.sql*' -o -name '*.db*' -o -name '*.bak*'")) + +print("\nCheck Notification in DB:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT id, name, type, is_default FROM notification;'")) + +c.close() + diff --git a/scripts/check_kuma_details.py b/scripts/check_kuma_details.py new file mode 100644 index 0000000..670206a --- /dev/null +++ b/scripts/check_kuma_details.py @@ -0,0 +1,23 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +print("Monitors count:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT count(id) FROM monitor;'")) + +print("\nStatus page count:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT * FROM status_page;'")) + +print("\nGroup count:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT * FROM `group`;'")) + +c.close() + diff --git a/scripts/check_kuma_live_status.py b/scripts/check_kuma_live_status.py new file mode 100644 index 0000000..e635b47 --- /dev/null +++ b/scripts/check_kuma_live_status.py @@ -0,0 +1,27 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("=== Uptime Kuma Pods ===") +print(run("kubectl get pods -n monitoring -o wide")) + +print("\n=== Uptime Kuma Deployment & Volumes ===") +print(run("kubectl get deployment -n monitoring uptime-kuma-2-1756668961 -o yaml | grep -A 10 volumes")) + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma-2-1756668961 -o jsonpath='{.items[0].metadata.name}'") +print(f"\n=== Current Pod: {pod} ===") + +print("\n=== Query MariaDB monitors table ===") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT count(*) FROM monitor; SELECT id, name, type, active FROM monitor;'")) + +print("\n=== Files in /app/data ===") +print(run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data")) + +c.close() + diff --git a/scripts/check_kuma_live_status2.py b/scripts/check_kuma_live_status2.py new file mode 100644 index 0000000..185e529 --- /dev/null +++ b/scripts/check_kuma_live_status2.py @@ -0,0 +1,25 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64" +print(f"=== Querying Pod: {pod} ===") +print("Query MariaDB monitor count:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT count(*) FROM monitor;'")) + +print("Query all monitors:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT id, name, type, active FROM monitor;'")) + +print("\nFiles in /app/data:") +print(run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data")) + +c.close() + diff --git a/scripts/check_kuma_logs.py b/scripts/check_kuma_logs.py new file mode 100644 index 0000000..fc5984c --- /dev/null +++ b/scripts/check_kuma_logs.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64" +print("=== Error log from /app/data/error.log ===") +print(run(f"kubectl exec -n monitoring {pod} -- tail -n 50 /app/data/error.log")) + +print("\n=== Recent pod logs ===") +print(run(f"kubectl logs -n monitoring {pod} --tail=60")) + +c.close() + diff --git a/scripts/check_kuma_mariadb.py b/scripts/check_kuma_mariadb.py new file mode 100644 index 0000000..f34edd6 --- /dev/null +++ b/scripts/check_kuma_mariadb.py @@ -0,0 +1,17 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-6656d4d5c6-z5cz9 -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT count(*) AS monitor_count FROM monitor; SELECT id, name, type, active FROM monitor LIMIT 15; SELECT * FROM notification;" """ +_, stdout, stderr = c.exec_command(cmd) +print(stdout.read().decode()) +err = stderr.read().decode() +if err: + print("[stderr]", err) + +c.close() + diff --git a/scripts/check_kuma_notif.py b/scripts/check_kuma_notif.py new file mode 100644 index 0000000..bd7a38f --- /dev/null +++ b/scripts/check_kuma_notif.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT * FROM notification;'")) + +c.close() + diff --git a/scripts/check_kuma_slugs.py b/scripts/check_kuma_slugs.py new file mode 100644 index 0000000..88ff619 --- /dev/null +++ b/scripts/check_kuma_slugs.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print("group:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'DESCRIBE `group`;'")) +print("\nmonitor_group:") +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'DESCRIBE monitor_group;'")) + +c.close() diff --git a/scripts/check_kuma_sp_schema.py b/scripts/check_kuma_sp_schema.py new file mode 100644 index 0000000..4aae54d --- /dev/null +++ b/scripts/check_kuma_sp_schema.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +print("--- STATUS PAGE SCHEMA ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE status_page; DESCRIBE \`group\`; DESCRIBE monitor_group;" """)) + +print("\n--- CURRENT GROUPS ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM \`group\`;" """)) + +c.close() + diff --git a/scripts/check_live_cm.py b/scripts/check_live_cm.py new file mode 100644 index 0000000..2405f5b --- /dev/null +++ b/scripts/check_live_cm.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('kubectl get cm homepage-2-1744068158 -n infrastructure -o yaml') +cm_yaml = o.read().decode() +start = cm_yaml.find('settings.yaml:') +print(cm_yaml[start:start+400]) + +c.close() + diff --git a/scripts/check_longhorn_source.py b/scripts/check_longhorn_source.py new file mode 100644 index 0000000..d13312c --- /dev/null +++ b/scripts/check_longhorn_source.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Pod:', pod) +print('Search for longhorn in /app/src or /app:') +print(run(f'kubectl exec -n infrastructure {pod} -- grep -rn "Missing Longhorn URL" /app')) + +c.close() + diff --git a/scripts/check_longhorn_svc.py b/scripts/check_longhorn_svc.py new file mode 100644 index 0000000..5aecde9 --- /dev/null +++ b/scripts/check_longhorn_svc.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +print("Spaceport longhorn-system services:") +print(run("kubectl get svc -n longhorn-system")) + +print("\nSpaceport ingress:") +print(run("kubectl get ingress -n longhorn-system")) + +c.close() + diff --git a/scripts/check_mdi_icons.py b/scripts/check_mdi_icons.py new file mode 100644 index 0000000..5c5d38f --- /dev/null +++ b/scripts/check_mdi_icons.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") + +print("Checking mdi weather in @mdi/js:") +print(run(f"kubectl exec -n infrastructure {pod} -- node -e \"const mdi = require('@mdi/js'); console.log('mdiWeatherStation:', !!mdi.mdiWeatherStation); console.log('mdiWeatherPartlyCloudy:', !!mdi.mdiWeatherPartlyCloudy);\"")) + +c.close() + diff --git a/scripts/check_monitoring_ns.py b/scripts/check_monitoring_ns.py new file mode 100644 index 0000000..9064ee8 --- /dev/null +++ b/scripts/check_monitoring_ns.py @@ -0,0 +1,17 @@ +import paramiko + +def check_ns(ip, name): + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print(f"=== {name} ({ip}) ===") + _, o, _ = c.exec_command("kubectl get all -n monitoring") + print(o.read().decode()) + if name == 'Aerodrome': + _, o, _ = c.exec_command("kubectl get all -n cattle-monitoring-system") + print("cattle-monitoring-system:\n", o.read().decode()) + c.close() + +check_ns('140.44.4.11', 'Aerodrome') +check_ns('140.44.4.41', 'Spaceport') + diff --git a/scripts/check_monitoring_spaceport.py b/scripts/check_monitoring_spaceport.py new file mode 100644 index 0000000..5e659c6 --- /dev/null +++ b/scripts/check_monitoring_spaceport.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl get all -n monitoring")) +c.close() + diff --git a/scripts/check_nas_backups.py b/scripts/check_nas_backups.py new file mode 100644 index 0000000..1b128d9 --- /dev/null +++ b/scripts/check_nas_backups.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Check /mnt/nfs_shares/backups:") +print(run("ls -la /mnt/nfs_shares/backups")) + +print("Create /mnt/nfs_shares/game-backups:") +print(run("mkdir -p /mnt/nfs_shares/game-backups && ls -la /mnt/nfs_shares/game-backups")) + +c.close() + diff --git a/scripts/check_nas_ports.py b/scripts/check_nas_ports.py new file mode 100644 index 0000000..cfb7747 --- /dev/null +++ b/scripts/check_nas_ports.py @@ -0,0 +1,12 @@ +import socket + +for ip in ['14.10.10.71', '14.10.10.72']: + print(f"\nChecking ports on {ip}:") + for port in [22, 80, 111, 161, 443, 445, 2049, 5000, 5001]: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1.0) + res = s.connect_ex((ip, port)) + if res == 0: + print(f" Port {port} OPEN") + s.close() + diff --git a/scripts/check_nvidia_host.py b/scripts/check_nvidia_host.py new file mode 100644 index 0000000..9f686b7 --- /dev/null +++ b/scripts/check_nvidia_host.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'") + return stdout.read().decode().strip() + +print("Check nvidia-smi:") +print(run("which nvidia-smi || nvidia-smi")) +print("Check GPU devices:") +print(run("ls -l /dev/nvidia*")) + +c.close() + diff --git a/scripts/check_nvidia_output.py b/scripts/check_nvidia_output.py new file mode 100644 index 0000000..8b6f6fc --- /dev/null +++ b/scripts/check_nvidia_output.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'") + return stdout.read().decode().strip() + +print("nvidia-smi output:") +print(run("/usr/local/bin/nvidia-smi")) + +c.close() + diff --git a/scripts/check_pkg_icons.py b/scripts/check_pkg_icons.py new file mode 100644 index 0000000..778d099 --- /dev/null +++ b/scripts/check_pkg_icons.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") + +print("Checking package.json dependencies:") +print(run(f"kubectl exec -n infrastructure {pod} -- grep -E 'mdi|icon|simple' /app/package.json")) + +c.close() + diff --git a/scripts/check_pod_logs.py b/scripts/check_pod_logs.py new file mode 100644 index 0000000..0de851c --- /dev/null +++ b/scripts/check_pod_logs.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Pod:', pod) +print('Recent pod logs:') +print(run(f'kubectl logs -n infrastructure {pod} --tail=50')) +c.close() + diff --git a/scripts/check_ports.py b/scripts/check_ports.py new file mode 100644 index 0000000..7ceaa21 --- /dev/null +++ b/scripts/check_ports.py @@ -0,0 +1,24 @@ +import socket + +hosts = [ + ('140.44.4.11', 9100, 'Aerodrome Master 01'), + ('140.44.4.41', 9100, 'Spaceport Master 01'), + ('140.44.4.71', 9100, 'Shipyard Swarm'), + ('140.44.4.81', 9100, 'Harvester Node 01'), + ('140.44.4.81', 9400, 'Harvester Node 01 DCGM'), + ('140.44.4.11', 10250, 'Aerodrome Kubelet'), + ('140.44.4.41', 10250, 'Spaceport Kubelet'), + ('140.44.4.60', 8888, 'Zabbix Web'), + ('140.44.4.64', 10051, 'Zabbix Server'), +] + +for ip, port, desc in hosts: + s = socket.socket() + s.settimeout(1.5) + try: + s.connect((ip, port)) + print(f"[OPEN] {ip}:{port} ({desc})") + s.close() + except Exception as e: + print(f"[CLOSED/ERR] {ip}:{port} ({desc}) - {e}") + diff --git a/scripts/check_prom_elsewhere.py b/scripts/check_prom_elsewhere.py new file mode 100644 index 0000000..16f8858 --- /dev/null +++ b/scripts/check_prom_elsewhere.py @@ -0,0 +1,14 @@ +import paramiko + +def check_host(ip, name, cmd): + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print(f"=== {name} ({ip}) ===") + _, o, _ = c.exec_command(cmd) + print(o.read().decode()) + c.close() + +check_host('140.44.4.71', 'Swarm', 'docker service ls; docker ps | grep -i prom') +check_host('140.44.44.121', 'Spacedock', 'kubectl get pods,svc -A | grep -i prom') + diff --git a/scripts/check_prom_live_targets.py b/scripts/check_prom_live_targets.py new file mode 100644 index 0000000..00ec97e --- /dev/null +++ b/scripts/check_prom_live_targets.py @@ -0,0 +1,29 @@ +import paramiko +import json + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +out = run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets") +data = json.loads(out) +print("=== ACTIVE TARGETS ===") +for t in data.get('data', {}).get('activeTargets', []): + print(f"[{t.get('health')}] pool: {t.get('scrapePool')} | job: {t.get('labels', {}).get('job')} | instance: {t.get('labels', {}).get('instance')} | url: {t.get('scrapeUrl')}") + if t.get('lastError'): + print(f" ERROR: {t.get('lastError')}") + +print("\n=== SAMPLE NODE EXPORTER QUERY ===") +out = run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(node_cpu_seconds_total{mode=\"idle\"})_by_(instance)'") +print(out) + +print("\n=== SAMPLE CADVISOR QUERY ===") +out = run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(container_cpu_usage_seconds_total)'") +print(out) + +c.close() + diff --git a/scripts/check_prom_metrics.py b/scripts/check_prom_metrics.py new file mode 100644 index 0000000..17eaca3 --- /dev/null +++ b/scripts/check_prom_metrics.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("--- PROMETHEUS TARGETS ---") +print(run("curl -s http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | \"\\(.labels.job) -> \\(.scrapeUrl) (\\(.health))\"' | head -n 30")) + +print("\n--- SAMPLE NODE METRICS ---") +print(run("curl -s 'http://localhost:9090/api/v1/query?query=node_cpu_seconds_total' | jq -r '.data.result | length'")) + +print("\n--- SAMPLE CONTAINER METRICS ---") +print(run("curl -s 'http://localhost:9090/api/v1/query?query=container_cpu_usage_seconds_total' | jq -r '.data.result | length'")) + +c.close() + diff --git a/scripts/check_prom_pods.py b/scripts/check_prom_pods.py new file mode 100644 index 0000000..4a540ab --- /dev/null +++ b/scripts/check_prom_pods.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl get pods -A | grep -E 'prom|grafana|node-exporter|cadvisor'")) +c.close() + diff --git a/scripts/check_prom_sa.py b/scripts/check_prom_sa.py new file mode 100644 index 0000000..d856365 --- /dev/null +++ b/scripts/check_prom_sa.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Prometheus Deployment / ServiceAccount:") +print(run("kubectl get deployment prometheus -o yaml | grep -E 'serviceAccount|image'")) + +print("\nClusterRoles / RoleBindings for prometheus:") +print(run("kubectl get clusterrolebindings | grep -i prom")) + +c.close() + diff --git a/scripts/check_prom_targets.py b/scripts/check_prom_targets.py new file mode 100644 index 0000000..7990c8f --- /dev/null +++ b/scripts/check_prom_targets.py @@ -0,0 +1,19 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Prometheus pod status:") +print(run("kubectl get pods -l app.kubernetes.io/name=prometheus")) + +print("\nPrometheus targets:") +print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets")) + +c.close() + diff --git a/scripts/check_prom_targets2.py b/scripts/check_prom_targets2.py new file mode 100644 index 0000000..20907ce --- /dev/null +++ b/scripts/check_prom_targets2.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets")) +c.close() + diff --git a/scripts/check_prometheus.py b/scripts/check_prometheus.py new file mode 100644 index 0000000..fea5221 --- /dev/null +++ b/scripts/check_prometheus.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Aerodrome Prometheus pods & svc:") +print(run("kubectl get pods,svc -A | grep -i prometheus")) + +c.close() + diff --git a/scripts/check_prometheus_spaceport.py b/scripts/check_prometheus_spaceport.py new file mode 100644 index 0000000..582b1a6 --- /dev/null +++ b/scripts/check_prometheus_spaceport.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Spaceport namespaces:") +print(run("kubectl get ns")) +print("Spaceport Prometheus:") +print(run("kubectl get pods,svc -A | grep -i prometheus")) + +c.close() + diff --git a/scripts/check_real_gotify.py b/scripts/check_real_gotify.py new file mode 100644 index 0000000..aad2c33 --- /dev/null +++ b/scripts/check_real_gotify.py @@ -0,0 +1,24 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """import sqlite3 +conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db') +cur = conn.cursor() +print('=== REAL GOTIFY DB APPLICATIONS ===') +cols = [c[1] for c in cur.execute('PRAGMA table_info(applications)').fetchall()] +print('Cols:', cols) +for r in cur.execute('SELECT * FROM applications').fetchall(): + print(r) +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/check_real_gotify.py && python3 /tmp/check_real_gotify.py") +stdin.write(py_script) +stdin.close() +print(stdout.read().decode()) +print(stderr.read().decode()) + +c.close() + diff --git a/scripts/check_running_mounts.py b/scripts/check_running_mounts.py new file mode 100644 index 0000000..3df281e --- /dev/null +++ b/scripts/check_running_mounts.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl get pod {pod} -o jsonpath='{{.spec.containers[0].volumeMounts}}'")) + +c.close() + diff --git a/scripts/check_sc_aerodrome.py b/scripts/check_sc_aerodrome.py new file mode 100644 index 0000000..0629f8f --- /dev/null +++ b/scripts/check_sc_aerodrome.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Storage classes on Aerodrome:") +print(run("kubectl get sc")) + +c.close() + diff --git a/scripts/check_shepherd.py b/scripts/check_shepherd.py new file mode 100644 index 0000000..d4beabb --- /dev/null +++ b/scripts/check_shepherd.py @@ -0,0 +1,19 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, _ = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + +run("docker service ps shepherd_shepherd") +run("docker service logs --tail 30 shepherd_shepherd") +c.close() + diff --git a/scripts/check_spacedock_prom.py b/scripts/check_spacedock_prom.py new file mode 100644 index 0000000..23f9a08 --- /dev/null +++ b/scripts/check_spacedock_prom.py @@ -0,0 +1,24 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.44.121", username="rancher", password="Katzir476!", timeout=10) +chan = c.invoke_shell() +time.sleep(2) +if chan.recv_ready(): + chan.recv(4096) + +def run(cmd): + chan.send(cmd + "\r\n") + time.sleep(3) + out = "" + while chan.recv_ready(): + out += chan.recv(8192).decode(errors='ignore') + return out + +print("Spacedock / Harvester pods with prom:") +print(run("sudo kubectl get pods,svc -A | grep -i prom")) + +c.close() + diff --git a/scripts/check_status_page.py b/scripts/check_status_page.py new file mode 100644 index 0000000..17f06f7 --- /dev/null +++ b/scripts/check_status_page.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-8c5dc549c-jfwhc -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, slug, title, published FROM status_page;" """ +_, stdout, _ = c.exec_command(cmd) +print(stdout.read().decode()) +c.close() + diff --git a/scripts/check_status_page_schema.py b/scripts/check_status_page_schema.py new file mode 100644 index 0000000..9d2f2f3 --- /dev/null +++ b/scripts/check_status_page_schema.py @@ -0,0 +1,13 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-8c5dc549c-jfwhc -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE status_page; DESCRIBE status_page_cname;" """ +_, stdout, _ = c.exec_command(cmd) +print(stdout.read().decode()) +c.close() + diff --git a/scripts/check_storage_and_drives.py b/scripts/check_storage_and_drives.py new file mode 100644 index 0000000..f7d2cb8 --- /dev/null +++ b/scripts/check_storage_and_drives.py @@ -0,0 +1,58 @@ +import paramiko, json + +# 1. Check Spaceport & Aerodrome disks +for name, ip in [('Aerodrome', '140.44.4.11'), ('Spaceport', '140.44.4.41')]: + try: + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + _, o, _ = c.exec_command('kubectl get nodes.longhorn.io -n longhorn-system -o json') + data = json.loads(o.read().decode()) + total_max = 0 + total_avail = 0 + num_disks = 0 + for n in data['items']: + disks = n.get('status', {}).get('diskStatus', {}) + for d_name, d_val in disks.items(): + num_disks += 1 + total_max += d_val.get('storageMaximum', 0) + total_avail += d_val.get('storageAvailable', 0) + + tib = total_max / (1024**4) + gib = total_max / (1024**3) + avail_gib = total_avail / (1024**3) + print(f"{name} Longhorn -> Disks: {num_disks}, Total Storage: {tib:.2f} TiB ({gib:.0f} GiB), Available: {avail_gib:.0f} GiB") + c.close() + except Exception as e: + print(f"{name} -> error: {e}") + +# 2. Query Zabbix for NAS storage & drives +import urllib.request, ssl +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php" +def zbx(method, params, auth=None): + payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1} + headers = {"Content-Type": "application/json-rpc"} + if auth: + headers["Authorization"] = f"Bearer {auth}" + req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + resp = json.loads(r.read().decode()) + return resp.get("result", {}) + +try: + token = zbx("user.login", {"username": "Admin", "password": "zabbix"}) + hosts = zbx("host.get", {"filter": {"name": ["SN-NAS (DS920+)", "SnarfNAS (DS1821+)", "SN-NAS", "SnarfNAS"]}}, token) + print("\nZabbix NAS Hosts:") + for h in hosts: + print(h["hostid"], h["host"], h["name"]) + # Get items for storage / disks + items = zbx("item.get", {"hostids": h["hostid"], "search": {"name": "space"}}, token) + for it in items[:5]: + print(" ", it["name"], it.get("lastvalue")) +except Exception as e: + print("Zabbix error:", e) + diff --git a/scripts/check_swarm_networks.py b/scripts/check_swarm_networks.py new file mode 100644 index 0000000..35ecf37 --- /dev/null +++ b/scripts/check_swarm_networks.py @@ -0,0 +1,21 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("docker network ls") +run("docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.Networks}}'") + +c.close() + diff --git a/scripts/check_swarm_services_all.py b/scripts/check_swarm_services_all.py new file mode 100644 index 0000000..f8d0a49 --- /dev/null +++ b/scripts/check_swarm_services_all.py @@ -0,0 +1,14 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('docker ps --format "{{.Names}}\t{{.Image}}\t{{.Ports}}"') +print(o.read().decode()) + +_, o2, _ = c.exec_command('docker service ls') +print(o2.read().decode()) + +c.close() + diff --git a/scripts/check_tmp_file.py b/scripts/check_tmp_file.py new file mode 100644 index 0000000..e4ff796 --- /dev/null +++ b/scripts/check_tmp_file.py @@ -0,0 +1,11 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('grep -A 20 "widgets.yaml:" /tmp/homepage.yaml') +print(o.read().decode()) + +c.close() + diff --git a/scripts/check_unhealthy_pods.py b/scripts/check_unhealthy_pods.py new file mode 100644 index 0000000..14a16d6 --- /dev/null +++ b/scripts/check_unhealthy_pods.py @@ -0,0 +1,16 @@ +import paramiko + +def check(ip, name): + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print(f"=== {name} ({ip}) Non-running Pods ===") + _, o, _ = c.exec_command("kubectl get pods -A --no-headers | grep -v -E 'Running|Completed'") + res = o.read().decode().strip() + print(res if res else "All pods are Running/Completed!") + c.close() + +check('140.44.4.11', 'Aerodrome') +print() +check('140.44.4.41', 'Spaceport') + diff --git a/scripts/check_unifi_client.py b/scripts/check_unifi_client.py new file mode 100644 index 0000000..bb4a796 --- /dev/null +++ b/scripts/check_unifi_client.py @@ -0,0 +1,40 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +s = requests.Session() +login_url = "https://140.44.44.1/api/auth/login" +login_payload = {"username": "Snarf", "password": "Katzir476!!!"} + +try: + r = s.post(login_url, json=login_payload, verify=False, timeout=5) + print("UniFi login:", r.status_code) + + # Query clients + r = s.get("https://140.44.44.1/proxy/network/api/s/default/stat/sta", verify=False, timeout=5) + if r.status_code == 200: + clients = r.json().get("data", []) + print(f"Total active clients: {len(clients)}") + target_mac = "bc:24:11:67:16:cd" + found = False + for c in clients: + if c.get("mac") == target_mac: + print("FOUND CLIENT:", json.dumps(c, indent=2)) + found = True + break + if not found: + print(f"Client {target_mac} not active. Checking configured/known clients...") + r2 = s.get("https://140.44.44.1/proxy/network/api/s/default/rest/user", verify=False, timeout=5) + if r2.status_code == 200: + known = r2.json().get("data", []) + for k in known: + if k.get("mac") == target_mac: + print("FOUND IN KNOWN USERS:", json.dumps(k, indent=2)) + found = True + break + if not found: + print(f"MAC {target_mac} not found in UniFi.") +except Exception as e: + print("UniFi error:", e) + diff --git a/scripts/check_zabbix_templates.py b/scripts/check_zabbix_templates.py new file mode 100644 index 0000000..16c71db --- /dev/null +++ b/scripts/check_zabbix_templates.py @@ -0,0 +1,41 @@ +import json +import ssl +import urllib.request + +ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php" +ZABBIX_USER = "Admin" +ZABBIX_PASS = "zabbix" + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +def zbx_call(method, params, auth=None): + payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1} + headers = {"Content-Type": "application/json-rpc"} + if auth: + headers["Authorization"] = f"Bearer {auth}" + req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + resp = json.loads(r.read().decode("utf-8")) + if "error" in resp: + raise RuntimeError(f"Zabbix error in {method}: {resp['error']}") + return resp["result"] + +auth_token = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS}) + +# Get host groups +groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth_token) +print("=== Host Groups ===") +for g in groups: + print(f"ID: {g['groupid']}, Name: {g['name']}") + +# Get ICMP Ping template +templates = zbx_call("template.get", { + "output": ["templateid", "name"], + "filter": {"name": ["ICMP Ping", "Chassis by IPMI", "Dell iDRAC by SNMP", "Ubiquiti UniFi by SNMP"]} +}, auth_token) +print("\n=== Templates ===") +for t in templates: + print(f"ID: {t['templateid']}, Name: {t['name']}") + diff --git a/scripts/create_argo_token.py b/scripts/create_argo_token.py new file mode 100644 index 0000000..8c43c69 --- /dev/null +++ b/scripts/create_argo_token.py @@ -0,0 +1,13 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +url = "https://argo.snarfnet.net/api/v1/session" +r = requests.post(url, json={"username": "admin", "password": "BbfHh3B-j6aHg3A-"}, verify=False) +admin_tok = r.json().get('token') +headers = {"Authorization": f"Bearer {admin_tok}"} + +tok_url = "https://argo.snarfnet.net/api/v1/account/homepage/token" +r_tok = requests.post(tok_url, headers=headers, json={"expiresIn": 0}, verify=False) +print("ArgoCD Token:", r_tok.json().get('token')) diff --git a/scripts/deploy_heimdall_fixed.py b/scripts/deploy_heimdall_fixed.py new file mode 100644 index 0000000..b69dc71 --- /dev/null +++ b/scripts/deploy_heimdall_fixed.py @@ -0,0 +1,66 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +new_compose = '''version: "3.8" + +services: + heimdall: + image: lscr.io/linuxserver/heimdall:latest + ports: + - target: 80 + published: 85 + mode: ingress + - target: 443 + published: 403 + mode: ingress + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/New_York + volumes: + - type: bind + source: /var/lib/docker-data/heimdall/config + target: /config + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - node.hostname == snshpd4lnxswm01 + restart_policy: + condition: on-failure +''' + +sftp = c.open_sftp() +with sftp.file('/tmp/heimdall_compose.yml', 'w') as f: + f.write(new_compose) +sftp.close() + +# Also save into /var/lib/docker-data/heimdall/compose.yml for persistence +run("cp /tmp/heimdall_compose.yml /var/lib/docker-data/heimdall/compose.yml") + +# Redeploy +run("docker stack deploy -c /tmp/heimdall_compose.yml heimdall") +time.sleep(6) +run("docker stack ps heimdall") +run("curl -s -I http://127.0.0.1:85 | head -n 5") + +c.close() + diff --git a/scripts/deploy_node_exporter.py b/scripts/deploy_node_exporter.py new file mode 100644 index 0000000..652e1b4 --- /dev/null +++ b/scripts/deploy_node_exporter.py @@ -0,0 +1,72 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +test_script = """ +cat << 'EOF' | kubectl apply -f - +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: kube-system + labels: + app.kubernetes.io/name: node-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: node-exporter + template: + metadata: + labels: + app.kubernetes.io/name: node-exporter + spec: + hostNetwork: true + hostPID: true + tolerations: + - operator: "Exists" + containers: + - name: node-exporter + image: prom/node-exporter:v1.8.2 + args: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/host/root" + - "--web.listen-address=0.0.0.0:9100" + ports: + - name: metrics + containerPort: 9100 + hostPort: 9100 + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: root + hostPath: + path: / +EOF + +kubectl rollout status daemonset/node-exporter -n kube-system --timeout=60s +""" + +print(run(test_script)) +c.close() + diff --git a/scripts/deploy_node_exporter_spaceport.py b/scripts/deploy_node_exporter_spaceport.py new file mode 100644 index 0000000..6b7012c --- /dev/null +++ b/scripts/deploy_node_exporter_spaceport.py @@ -0,0 +1,72 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +test_script = """ +cat << 'EOF' | kubectl apply -f - +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: kube-system + labels: + app.kubernetes.io/name: node-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: node-exporter + template: + metadata: + labels: + app.kubernetes.io/name: node-exporter + spec: + hostNetwork: true + hostPID: true + tolerations: + - operator: "Exists" + containers: + - name: node-exporter + image: prom/node-exporter:v1.8.2 + args: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/host/root" + - "--web.listen-address=0.0.0.0:9100" + ports: + - name: metrics + containerPort: 9100 + hostPort: 9100 + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: root + hostPath: + path: / +EOF + +kubectl rollout status daemonset/node-exporter -n kube-system --timeout=60s +""" + +print(run(test_script)) +c.close() + diff --git a/scripts/deploy_shepherd.py b/scripts/deploy_shepherd.py new file mode 100644 index 0000000..61a2ef0 --- /dev/null +++ b/scripts/deploy_shepherd.py @@ -0,0 +1,63 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +print("=== 1. Removing Old Failing Watchtower Stack ===") +run("docker stack rm watchtower") +time.sleep(5) + +shepherd_compose = '''version: "3.8" + +services: + shepherd: + image: containrrr/shepherd:latest + environment: + - SLEEP_TIME=24h + - BLACKLIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup + - UPDATE_OPTIONS=--with-registry-auth + - ROLLBACK_FAILURE=true + - TZ=America/New_York + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - node.role == manager + restart_policy: + condition: on-failure +''' + +sftp = c.open_sftp() +with sftp.file('/tmp/shepherd_compose.yml', 'w') as f: + f.write(shepherd_compose) +sftp.close() + +print("\n=== 2. Deploying Swarm-Native Shepherd Stack ===") +run("docker stack deploy -c /tmp/shepherd_compose.yml shepherd") +time.sleep(6) + +run("docker stack ls") +run("docker stack ps shepherd") +run("docker service logs --tail 20 $(docker service ls -q -f name=shepherd) 2>/dev/null || true") + +c.close() +print("\n=== Shepherd Deployment Complete ===") + diff --git a/scripts/find_all_prom.py b/scripts/find_all_prom.py new file mode 100644 index 0000000..94577b9 --- /dev/null +++ b/scripts/find_all_prom.py @@ -0,0 +1,14 @@ +import paramiko + +def find_prom(ip, name): + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print(f"=== {name} ({ip}) ===") + _, o, _ = c.exec_command("kubectl get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}/status: {.metadata.name}{ \"\\t\" }{.spec.containers[*].image}{\"\\n\"}{end}' | grep -i prom") + print(o.read().decode()) + c.close() + +find_prom('140.44.4.11', 'Aerodrome') +find_prom('140.44.4.41', 'Spaceport') + diff --git a/scripts/find_all_weather_entities.py b/scripts/find_all_weather_entities.py new file mode 100644 index 0000000..70a1d3b --- /dev/null +++ b/scripts/find_all_weather_entities.py @@ -0,0 +1,26 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +ha_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4" + +cmd = f"""curl -s -H "Authorization: Bearer {ha_token}" -H "Content-Type: application/json" http://14.4.4.20:8123/api/states""" +_, stdout, stderr = c.exec_command(cmd) +raw = stdout.read().decode() + +try: + states = json.loads(raw) + weather_station_entities = [s for s in states if any(k in s['entity_id'].lower() for k in ['back_yard', 'weather_station', 'tempest', 'rain', 'wind', 'humidity'])] + print(f"\nFound {len(weather_station_entities)} matching entities:") + for s in weather_station_entities: + print(f" - {s['entity_id']} = {s['state']} (unit: {s.get('attributes', {}).get('unit_of_measurement')}, name: {s.get('attributes', {}).get('friendly_name')})") +except Exception as e: + print("Error:", e) + +c.close() + diff --git a/scripts/find_ha_api.py b/scripts/find_ha_api.py new file mode 100644 index 0000000..e76d49b --- /dev/null +++ b/scripts/find_ha_api.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print(run(f'kubectl exec -n infrastructure {pod} -- find /app/src/pages/api -name "*homeassistant*"')) +print('\nAlso search in /app/src/widgets or /app/src/utils:') +print(run(f'kubectl exec -n infrastructure {pod} -- grep -rn "homeassistant" /app/src/pages/api/ /app/src/widgets/')) + +c.close() diff --git a/scripts/find_ha_api2.py b/scripts/find_ha_api2.py new file mode 100644 index 0000000..701fd9b --- /dev/null +++ b/scripts/find_ha_api2.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Find in /app/src/pages/api:') +print(run(f'kubectl exec -n infrastructure {pod} -- ls -la /app/src/pages/api')) +print('Find in /app/src/widgets/homeassistant:') +print(run(f'kubectl exec -n infrastructure {pod} -- ls -la /app/src/widgets/homeassistant')) +print('Find widget.js or service.js:') +print(run(f'kubectl exec -n infrastructure {pod} -- find /app/src -name "widget.js*" -o -name "service.js*"')) + +c.close() diff --git a/scripts/find_ha_entities.py b/scripts/find_ha_entities.py new file mode 100644 index 0000000..09a12c4 --- /dev/null +++ b/scripts/find_ha_entities.py @@ -0,0 +1,28 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +ha_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4" + +cmd = f"""curl -s -H "Authorization: Bearer {ha_token}" -H "Content-Type: application/json" http://14.4.4.20:8123/api/states""" +_, stdout, stderr = c.exec_command(cmd) +raw = stdout.read().decode() + +try: + states = json.loads(raw) + print(f"Total HA entities: {len(states)}") + weatherflow_entities = [s for s in states if any(k in s['entity_id'].lower() for k in ['tempest', 'weatherflow', 'weather'])] + print(f"\nFound {len(weatherflow_entities)} Tempest / Weather entities:") + for s in weatherflow_entities: + print(f" - {s['entity_id']} = {s['state']} (name: {s.get('attributes', {}).get('friendly_name')})") +except Exception as e: + print("Error parsing HA response:", e) + print("Raw output preview:", raw[:500]) + +c.close() + diff --git a/scripts/find_homepage_icons.py b/scripts/find_homepage_icons.py new file mode 100644 index 0000000..eed7266 --- /dev/null +++ b/scripts/find_homepage_icons.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") + +print("Find icons matching argo:") +print(run(f"kubectl exec -n infrastructure {pod} -- find /app -iname '*argo*'")) + +print("\nFind icons matching weather or tempest:") +print(run(f"kubectl exec -n infrastructure {pod} -- find /app -iname '*weather*' -o -iname '*tempest*'")) + +c.close() + diff --git a/scripts/find_longhorn_config.py b/scripts/find_longhorn_config.py new file mode 100644 index 0000000..51ece3b --- /dev/null +++ b/scripts/find_longhorn_config.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Pod:', pod) +print('Files in /app/config:') +print(run(f'kubectl exec -n infrastructure {pod} -- ls -la /app/config')) +print('Grep longhorn in /app/config:') +print(run(f'kubectl exec -n infrastructure {pod} -- grep -rn "longhorn" /app/config/')) + +c.close() + diff --git a/scripts/find_nvml.py b/scripts/find_nvml.py new file mode 100644 index 0000000..796f5cf --- /dev/null +++ b/scripts/find_nvml.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'") + return stdout.read().decode().strip() + +print("Search for libnvidia-ml.so on host:") +print(run("find / -name 'libnvidia-ml.so*' 2>/dev/null")) + +c.close() + diff --git a/scripts/find_peanut_code.py b/scripts/find_peanut_code.py new file mode 100644 index 0000000..a51053b --- /dev/null +++ b/scripts/find_peanut_code.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app -name \"*peanut*\"")) +c.close() + diff --git a/scripts/find_proxy.py b/scripts/find_proxy.py new file mode 100644 index 0000000..7d219bb --- /dev/null +++ b/scripts/find_proxy.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Find proxy in /app/src:') +print(run(f'kubectl exec -n infrastructure {pod} -- find /app/src -name "*proxy*"')) + +c.close() diff --git a/scripts/find_proxy_files.py b/scripts/find_proxy_files.py new file mode 100644 index 0000000..fffd2ff --- /dev/null +++ b/scripts/find_proxy_files.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app/src -name \"*use-widget*\" -o -name \"*proxy*\"")) +c.close() + diff --git a/scripts/find_settings.py b/scripts/find_settings.py new file mode 100644 index 0000000..9a09535 --- /dev/null +++ b/scripts/find_settings.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print(run(f"kubectl exec -n infrastructure {pod} -- find /app -name '*setting*'")) + +c.close() + diff --git a/scripts/find_uptimekuma_files.py b/scripts/find_uptimekuma_files.py new file mode 100644 index 0000000..9f5bdd2 --- /dev/null +++ b/scripts/find_uptimekuma_files.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app -name \"*uptimekuma*\"")) +c.close() + diff --git a/scripts/find_utils.py b/scripts/find_utils.py new file mode 100644 index 0000000..02e703d --- /dev/null +++ b/scripts/find_utils.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app/src/utils -type f")) +c.close() + diff --git a/scripts/fix_gotify_sort_key.py b/scripts/fix_gotify_sort_key.py new file mode 100644 index 0000000..596bdb0 --- /dev/null +++ b/scripts/fix_gotify_sort_key.py @@ -0,0 +1,35 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """import sqlite3 +import shutil + +# 1. Backup +shutil.copyfile('/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db', '/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db.bak') + +# 2. Fix NULL sort_key in _data +conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db') +cur = conn.cursor() +cur.execute("UPDATE applications SET sort_key = 'a5' WHERE id = 6;") +conn.commit() +conn.close() + +# 3. Copy to parent directory where volume is bound +shutil.copyfile('/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db', '/mnt/swarm_shared/legacy_volumes/gotify/gotify.db') +print("Successfully updated sort_key and synced database!") +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/fix_gotify.py && python3 /tmp/fix_gotify.py") +stdin.write(py_script) +stdin.close() +print(stdout.read().decode()) +print(stderr.read().decode()) + +# Restart gotify service +print(c.exec_command("docker service update --force gotify_gotify")[1].read().decode()) + +c.close() + diff --git a/scripts/fix_homepage_icons.py b/scripts/fix_homepage_icons.py new file mode 100644 index 0000000..2751cfe --- /dev/null +++ b/scripts/fix_homepage_icons.py @@ -0,0 +1,42 @@ +import yaml + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + text = f.read() + +docs = list(yaml.safe_load_all(text)) +cm = docs[2] + +services = yaml.safe_load(cm['data']['services.yaml']) + +for group in services: + # 1. Fix ArgoCD icon in GitOps & Automation + if 'GitOps & Automation' in group: + for item in group['GitOps & Automation']: + if 'ArgoCD' in item: + print("Updating ArgoCD icon from:", item['ArgoCD'].get('icon')) + item['ArgoCD']['icon'] = 'argo-cd.png' + print("To:", item['ArgoCD']['icon']) + + # 2. Fix Tempest Weather Station icon in Environment & Weather + if 'Environment & Weather' in group: + for item in group['Environment & Weather']: + if 'Tempest Weather Station' in item: + print("Updating Tempest icon from:", item['Tempest Weather Station'].get('icon')) + item['Tempest Weather Station']['icon'] = 'mdi-weather-partly-cloudy' + print("To:", item['Tempest Weather Station']['icon']) + +cm['data']['services.yaml'] = yaml.dump(services, sort_keys=False, default_flow_style=False, allow_unicode=True) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f: + yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +print("\nSuccessfully updated homepage.yaml with corrected icons!") + diff --git a/scripts/fix_homepage_manifest.py b/scripts/fix_homepage_manifest.py new file mode 100644 index 0000000..764e268 --- /dev/null +++ b/scripts/fix_homepage_manifest.py @@ -0,0 +1,156 @@ +import os + +new_configmap_data = """apiVersion: v1 +data: + bookmarks.yaml: | + - Developer: + - Github: + - abbr: GH + href: https://github.com/ + docker.yaml: '' + kubernetes.yaml: | + mode: disable + services.yaml: | + - Critical Services: + - Authentik: + icon: authentik.png + href: https://authentik.snarfnet.net + description: SSO & Identity Management + - PeaNUT UPS: + icon: peanut.png + href: https://portal.snarfnet.net + description: UPS Power & Battery Monitor + - Unifi Controller: + icon: unifi.png + href: https://unifi.ui.com + widget: + type: unifi + url: https://140.44.44.1 + username: Snarf + password: Katzir476!!! + + - GitOps & Automation: + - Gitea: + icon: gitea.png + href: https://git.snarfnet.net + description: Git Repositories + - ArgoCD: + icon: argocd.png + href: https://argo.snarfnet.net + description: Declarative GitOps Reconciler + - AWX Automation: + icon: awx.png + href: https://awx.snarfnet.net + description: Ansible Automation Controller + - Semaphore: + icon: semaphore.png + href: https://semaphore.snarfnet.net + description: Playbook Execution Engine + + - Storage & Clusters: + - Longhorn Aerodrome: + icon: longhorn.png + href: https://longhorn.aerodrome.snarfnet.net + description: Aerodrome Distributed Flash Storage + - Longhorn Spaceport: + icon: longhorn.png + href: https://longhorn.spaceport.snarfnet.net + description: Spaceport Distributed Block Storage + - Rancher (Aerodrome): + icon: rancher.png + href: https://aerodrome.snarfnet.net + description: Kubernetes Management Console + - Harvester HCI (Spacedock): + icon: harvester.png + href: https://harvester.spacedock.snarfnet.net + description: HCI Virtualization Hypervisors + - Portainer Swarm: + icon: portainer.png + href: https://shipyard.snarfnet.net + description: Docker Swarm Container Fleet + + - Infrastructure & Observability: + - NGINX Proxy Manager: + icon: nginx-proxy-manager + href: https://nginx.snarfnet.net + description: Reverse Proxy & SSL Certificates + - Uptime Kuma: + icon: uptime-kuma + href: https://uptime.snarfnet.net + description: Multi-Endpoint Availability Probes + - Zabbix: + icon: zabbix.png + href: https://zabbix.snarfnet.net + description: Infrastructure Metrics & Alarms + - Grafana: + icon: grafana.png + href: https://grafana.snarfnet.net + description: Real-Time Metrics & AI Dashboards + - Core Node Glances: + icon: glances.png + href: http://140.44.4.11:61208 + description: Host Resource Monitor + + - Environment & Weather: + - Tempest Weather Station: + icon: weather.png + href: https://snarfhome.snarfnet.net + description: Backyard Weather Telemetry + widget: + type: homeassistant + url: http://14.4.4.20:8123 + key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4 + custom: + - state: sensor.st_00158555_temperature + label: Outdoor Temp + - state: sensor.st_00158555_feels_like + label: Feels Like + - state: sensor.st_00158555_humidity + label: Humidity + - state: sensor.st_00158555_wind_speed + label: Wind Speed + + settings.yaml: | + title: SnarfNet Operations & Telemetry + headerStyle: boxedWidgets + color: green + theme: dark + layout: + Critical Services: + GitOps & Automation: + Storage & Clusters: + Infrastructure & Observability: + Environment & Weather: + showStats: true + + widgets.yaml: | + - resources: + backend: resources + cpu: true + expanded: true + memory: true + - search: + provider: duckduckgo + target: _blank +""" + +filepath = r"homelab-gitops-repo\manifests\spaceport\homepage\homepage.yaml" +with open(filepath, "r", encoding="utf-8") as f: + text = f.read() + +parts = text.split("---\n") +if len(parts) == 1: + parts = text.split("---\r\n") + +cm_part = parts[2] +cm_kind_idx = cm_part.index("kind: ConfigMap") +cm_suffix = cm_part[cm_kind_idx:] + +new_cm_part = new_configmap_data + cm_suffix +parts[2] = new_cm_part + +new_full = "---\n".join(parts) +with open(filepath, "w", encoding="utf-8", newline="\r\n") as f: + f.write(new_full) +print("Updated homepage.yaml cleanly!") + diff --git a/scripts/generate_ai_dashboard.py b/scripts/generate_ai_dashboard.py new file mode 100644 index 0000000..a99da9b --- /dev/null +++ b/scripts/generate_ai_dashboard.py @@ -0,0 +1,279 @@ +import json + +dashboard = { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": True, + "hide": True, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": True, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": None, + "links": [], + "liveNow": False, + "panels": [ + { + "collapsed": False, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "āš” AI Stack Executive Overview & Cloud Cost Offsets", + "type": "row" + }, + { + "id": 1, + "title": "Prompt Tokens Processed (In)", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "sum(vllm:prompt_tokens_total)", "legendFormat": "Prompt Tokens", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": None }] }, + "unit": "locale" + } + }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" } + }, + { + "id": 2, + "title": "Generation Tokens Synthesized (Out)", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "sum(vllm:generation_tokens_total)", "legendFormat": "Gen Tokens", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": None }] }, + "unit": "locale" + } + }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center" } + }, + { + "id": 3, + "title": "Homelab Cloud API Savings ($USD)", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "(sum(vllm:prompt_tokens_total) * 0.00000025) + (sum(vllm:generation_tokens_total) * 0.00000100)", "legendFormat": "Net Savings", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": None }] }, + "unit": "currencyUSD" + } + }, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center" } + }, + { + "id": 4, + "title": "Active GPU Cluster Health", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "count(DCGM_FI_DEV_GPU_TEMP)", "legendFormat": "Tesla T4s Active", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": None }] }, + "unit": "short" + } + }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center" } + }, + { + "collapsed": False, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 200, + "title": "šŸŽ® Tesla T4 GPU Cluster Telemetry (Harvester HCI Spacedock)", + "type": "row" + }, + { + "id": 5, + "title": "GPU Core Utilization (%)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "{{Hostname}} ({{modelName}})", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20 }, + "unit": "percent", + "min": 0, + "max": 100 + } + } + }, + { + "id": 6, + "title": "VRAM Memory Allocated & Used", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "DCGM_FI_DEV_FB_USED * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Used", "refId": "A" }, + { "expr": "(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 1024 * 1024", "legendFormat": "{{Hostname}} VRAM Total", "refId": "B" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 }, + "unit": "bytes" + } + } + }, + { + "id": 7, + "title": "GPU Thermal Profile (Ā°C)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "{{Hostname}} (Ā°C)", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 }, + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": None }, + { "color": "yellow", "value": 65 }, + { "color": "red", "value": 80 } + ] + } + } + } + }, + { + "id": 8, + "title": "GPU Power Draw (Watts)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "{{Hostname}} (Watts)", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15 }, + "unit": "watt" + } + } + }, + { + "collapsed": False, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "id": 300, + "title": "šŸ§  vLLM Real-Time Inference Dynamics & Token Velocity", + "type": "row" + }, + { + "id": 9, + "title": "Token Generation Velocity (Tokens / Sec)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "sum(rate(vllm:generation_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (Out)", "refId": "A" }, + { "expr": "sum(rate(vllm:prompt_tokens_total[2m])) by (model_name)", "legendFormat": "{{model_name}} (In)", "refId": "B" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 25 }, + "unit": "cps" + } + } + }, + { + "id": 10, + "title": "Inference Concurrency & Request Backlog", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "vllm:num_requests_running", "legendFormat": "{{model_name}} Running", "refId": "A" }, + { "expr": "vllm:num_requests_waiting", "legendFormat": "{{model_name}} Queued", "refId": "B" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "stepAfter", "fillOpacity": 15 }, + "unit": "short" + } + } + }, + { + "id": 11, + "title": "GPU KV Cache Memory Saturation", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "vllm:gpu_cache_usage_factor * 100", "legendFormat": "{{model_name}} Cache Usage", "refId": "A" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 30 }, + "unit": "percent", + "min": 0, + "max": 100 + } + } + }, + { + "id": 12, + "title": "NVIDIA Triton & Qdrant Operations", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 }, + "datasource": "Prometheus", + "targets": [ + { "expr": "rate(nv_inference_request_success[2m])", "legendFormat": "Triton Success/s", "refId": "A" }, + { "expr": "cpu_cores_used", "legendFormat": "Qdrant CPU Cores", "refId": "B" } + ], + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth" }, + "unit": "short" + } + } + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["ai", "vllm", "dcgm", "gpu", "snarfnet"], + "templating": { "list": [] }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m"] + }, + "timezone": "browser", + "title": "SnarfNet AI Stack & Inference Intelligence", + "uid": "snarfnet-ai-stack", + "version": 1 +} + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/ai-stack-dashboard.json", "w") as f: + json.dump(dashboard, f, indent=2) + +print("Created ai-stack-dashboard.json successfully!") + diff --git a/scripts/generate_systems_container_dashboards.py b/scripts/generate_systems_container_dashboards.py new file mode 100644 index 0000000..27b9527 --- /dev/null +++ b/scripts/generate_systems_container_dashboards.py @@ -0,0 +1,246 @@ +import json + +def make_panel_stat(id_val, title, expr, grid_pos, unit="none", color="green"): + return { + "id": id_val, + "title": title, + "type": "stat", + "gridPos": grid_pos, + "datasource": "Prometheus", + "targets": [{"expr": expr, "legendFormat": "{{instance}}", "refId": "A"}], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"mode": "absolute", "steps": [{"color": color, "value": None}]}, + "unit": unit + } + }, + "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"} + } + +def make_panel_timeseries(id_val, title, targets, grid_pos, unit="none"): + return { + "id": id_val, + "title": title, + "type": "timeseries", + "gridPos": grid_pos, + "datasource": "Prometheus", + "targets": targets, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15 + }, + "unit": unit + } + }, + "options": {"tooltip": {"mode": "multi", "sort": "desc"}} + } + +def make_panel_bargauge(id_val, title, expr, grid_pos, unit="percent"): + return { + "id": id_val, + "title": title, + "type": "bargauge", + "gridPos": grid_pos, + "datasource": "Prometheus", + "targets": [{"expr": expr, "legendFormat": "{{instance}}", "refId": "A"}], + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 75}, + {"color": "red", "value": 90} + ] + }, + "unit": unit, + "min": 0, + "max": 100 + } + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": True + } + } + +# ==================== 1. SYSTEMS DASHBOARD ==================== +systems_dashboard = { + "annotations": {"list": []}, + "editable": True, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": None, + "links": [], + "liveNow": False, + "panels": [ + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 100, + "title": "šŸ–„ļø Systems Fleet Overview & Capacity", + "type": "row" + }, + make_panel_stat(1, "Online Node Exporters", "count(node_load1)", {"h": 4, "w": 6, "x": 0, "y": 1}, "none", "green"), + make_panel_stat(2, "Avg Cluster CPU Load (1m)", "avg(node_load1)", {"h": 4, "w": 6, "x": 6, "y": 1}, "none", "blue"), + make_panel_stat(3, "Total Monitored RAM", "sum(node_memory_MemTotal_bytes)", {"h": 4, "w": 6, "x": 12, "y": 1}, "bytes", "purple"), + make_panel_stat(4, "Fleet RAM In-Use %", "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", {"h": 4, "w": 6, "x": 18, "y": 1}, "percent", "orange"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, + "id": 200, + "title": "āš” CPU Load & Core Utilization", + "type": "row" + }, + make_panel_timeseries(5, "CPU Utilization by Host (%)", [ + {"expr": "100 - (avg by (instance, cluster) (rate(node_cpu_seconds_total{mode='idle'}[2m])) * 100)", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 0, "y": 6}, "percent"), + make_panel_timeseries(6, "1-Minute Load Average by Host", [ + {"expr": "node_load1", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 12, "y": 6}, "none"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}, + "id": 300, + "title": "šŸ§  Memory Consumption & Free Capacity", + "type": "row" + }, + make_panel_timeseries(7, "Memory Used % by Host", [ + {"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 0, "y": 15}, "percent"), + make_panel_timeseries(8, "Available RAM by Host (GiB)", [ + {"expr": "node_memory_MemAvailable_bytes", "legendFormat": "{{cluster}} - {{instance}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 12, "y": 15}, "bytes"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23}, + "id": 400, + "title": "šŸ’¾ Filesystem & Disk I/O Performance", + "type": "row" + }, + make_panel_bargauge(9, "Root (/) Filesystem Usage %", "(1 - (node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})) * 100", {"h": 8, "w": 8, "x": 0, "y": 24}, "percent"), + make_panel_timeseries(10, "Disk Read Throughput (Bytes/sec)", [ + {"expr": "sum by (instance) (rate(node_disk_read_bytes_total[2m]))", "legendFormat": "Read {{instance}}", "refId": "A"} + ], {"h": 8, "w": 8, "x": 8, "y": 24}, "Bps"), + make_panel_timeseries(11, "Disk Write Throughput (Bytes/sec)", [ + {"expr": "sum by (instance) (rate(node_disk_written_bytes_total[2m]))", "legendFormat": "Write {{instance}}", "refId": "A"} + ], {"h": 8, "w": 8, "x": 16, "y": 24}, "Bps"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 32}, + "id": 500, + "title": "šŸŒ Network Throughput (RX / TX)", + "type": "row" + }, + make_panel_timeseries(12, "Host Network Receive (RX)", [ + {"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", "legendFormat": "RX {{instance}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 0, "y": 33}, "Bps"), + make_panel_timeseries(13, "Host Network Transmit (TX)", [ + {"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~'lo|veth.*|flannel.*|cali.*'}[2m]))", "legendFormat": "TX {{instance}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 12, "y": 33}, "Bps") + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["infrastructure", "systems", "snarfnet", "node-exporter"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "šŸ–„ļø Systems & Cluster Host Performance", + "uid": "snarfnet-systems" +} + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/systems-dashboard.json", "w", encoding="utf-8") as f: + json.dump(systems_dashboard, f, indent=2) + +print("Created systems-dashboard.json successfully!") + +# ==================== 2. CONTAINER METRICS DASHBOARD ==================== +container_dashboard = { + "annotations": {"list": []}, + "editable": True, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": None, + "links": [], + "liveNow": False, + "panels": [ + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 100, + "title": "šŸ“¦ Container Fleet & Pod Metrics", + "type": "row" + }, + make_panel_stat(1, "Monitored Containers", "count(container_last_seen{container!=''})", {"h": 4, "w": 6, "x": 0, "y": 1}, "none", "green"), + make_panel_stat(2, "Active Kubernetes Pods", "count(count by (pod, namespace) (container_last_seen{container!=''}))", {"h": 4, "w": 6, "x": 6, "y": 1}, "none", "blue"), + make_panel_stat(3, "Total Container CPU Load", "sum(rate(container_cpu_usage_seconds_total{container!=''}[2m]))", {"h": 4, "w": 6, "x": 12, "y": 1}, "none", "purple"), + make_panel_stat(4, "Total Container RAM", "sum(container_memory_working_set_bytes{container!=''})", {"h": 4, "w": 6, "x": 18, "y": 1}, "bytes", "orange"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, + "id": 200, + "title": "āš” Top Container CPU Utilization", + "type": "row" + }, + make_panel_timeseries(5, "Top 10 CPU Consuming Pods (Cores)", [ + {"expr": "topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=''}[2m])))", "legendFormat": "{{namespace}} / {{pod}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 0, "y": 6}, "none"), + make_panel_timeseries(6, "CPU Throttling by Namespace", [ + {"expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total[2m]))", "legendFormat": "{{namespace}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 12, "y": 6}, "none"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}, + "id": 300, + "title": "šŸ§  Container Memory Working Set", + "type": "row" + }, + make_panel_timeseries(7, "Top 10 Memory Consuming Pods", [ + {"expr": "topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{container!=''}))", "legendFormat": "{{namespace}} / {{pod}}", "refId": "A"} + ], {"h": 8, "w": 14, "x": 0, "y": 15}, "bytes"), + make_panel_bargauge(8, "Memory by Namespace", "sum by (namespace) (container_memory_working_set_bytes{container!=''})", {"h": 8, "w": 10, "x": 14, "y": 15}, "bytes"), + + { + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23}, + "id": 400, + "title": "šŸŒ Container Network Traffic", + "type": "row" + }, + make_panel_timeseries(9, "Top Pod Network RX (Receive)", [ + {"expr": "topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total[2m])))", "legendFormat": "RX {{namespace}}/{{pod}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 0, "y": 24}, "Bps"), + make_panel_timeseries(10, "Top Pod Network TX (Transmit)", [ + {"expr": "topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total[2m])))", "legendFormat": "TX {{namespace}}/{{pod}}", "refId": "A"} + ], {"h": 8, "w": 12, "x": 12, "y": 24}, "Bps") + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["kubernetes", "containers", "cadvisor", "snarfnet", "workloads"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "šŸ“¦ Kubernetes Containers & Workloads", + "uid": "snarfnet-containers" +} + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/container-metrics-dashboard.json", "w", encoding="utf-8") as f: + json.dump(container_dashboard, f, indent=2) + +print("Created container-metrics-dashboard.json successfully!") + diff --git a/scripts/get_argo_token.py b/scripts/get_argo_token.py new file mode 100644 index 0000000..bee5df3 --- /dev/null +++ b/scripts/get_argo_token.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=argocd-server -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl exec -n infrastructure {pod} -- argocd account generate-token --account admin")) + +c.close() + diff --git a/scripts/get_gitea_token.py b/scripts/get_gitea_token.py new file mode 100644 index 0000000..2496bf0 --- /dev/null +++ b/scripts/get_gitea_token.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=gitea -o jsonpath='{.items[0].metadata.name}'") +print("Gitea pod:", pod) + +token = run(f"kubectl exec -n infrastructure {pod} -c gitea -- gitea admin user generate-access-token -u Snarf -t homepage-token --scopes all") +print("Generated token:", token) + +c.close() + diff --git a/scripts/get_zabbix_metadata.py b/scripts/get_zabbix_metadata.py new file mode 100644 index 0000000..75b49d1 --- /dev/null +++ b/scripts/get_zabbix_metadata.py @@ -0,0 +1,36 @@ +import json +import ssl +import urllib.request + +ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php" +ZABBIX_USER = "Admin" +ZABBIX_PASS = "zabbix" + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +def zbx_call(method, params, auth=None): + payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1} + headers = {"Content-Type": "application/json-rpc"} + if auth: + headers["Authorization"] = f"Bearer {auth}" + req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + resp = json.loads(r.read().decode("utf-8")) + if "error" in resp: + raise RuntimeError(f"Zabbix error in {method}: {resp['error']}") + return resp["result"] + +auth = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS}) + +groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth) +print("=== Host Groups ===") +for g in groups: + print(f" {g['groupid']}: {g['name']}") + +templates = zbx_call("template.get", {"output": ["templateid", "name"], "search": {"name": "Ping"}}, auth) +print("\n=== Ping Templates ===") +for t in templates: + print(f" {t['templateid']}: {t['name']}") + diff --git a/scripts/grep_icon_source.py b/scripts/grep_icon_source.py new file mode 100644 index 0000000..d8d632b --- /dev/null +++ b/scripts/grep_icon_source.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") + +print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'dashboard-icons' /app/.next/server/ | head -n 5")) +print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'walkxcode' /app/.next/server/ | head -n 5")) + +c.close() + diff --git a/scripts/inspect_85776.py b/scripts/inspect_85776.py new file mode 100644 index 0000000..f351179 --- /dev/null +++ b/scripts/inspect_85776.py @@ -0,0 +1,26 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('azuredevops'); +console.log(content.substring(idx, idx + 1200)); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/check_85776_3.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/check_85776_3.js")) + +c.close() diff --git a/scripts/inspect_api_pages.py b/scripts/inspect_api_pages.py new file mode 100644 index 0000000..a755ffd --- /dev/null +++ b/scripts/inspect_api_pages.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app/src/pages/api -type f")) +c.close() + diff --git a/scripts/inspect_app_tree.py b/scripts/inspect_app_tree.py new file mode 100644 index 0000000..64fc677 --- /dev/null +++ b/scripts/inspect_app_tree.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app -maxdepth 3 -not -path \"*/node_modules*\" -not -path \"*/.next*\"")) +c.close() + diff --git a/scripts/inspect_customapi.py b/scripts/inspect_customapi.py new file mode 100644 index 0000000..ae13511 --- /dev/null +++ b/scripts/inspect_customapi.py @@ -0,0 +1,36 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print("=== /app/src/widgets/customapi/component.jsx ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/widgets/customapi/component.jsx")) + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('customapiProxyHandler'); +if (idx !== -1) { + console.log(content.substring(Math.max(0, idx - 100), Math.min(content.length, idx + 1000))); +} else { + const idx2 = content.indexOf('customapi'); + console.log('customapi at', idx2); + console.log(content.substring(Math.max(0, idx2 - 100), Math.min(content.length, idx2 + 1000))); +} +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/customapi_code.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/customapi_code.js")) + +c.close() + diff --git a/scripts/inspect_glances_proxy.py b/scripts/inspect_glances_proxy.py new file mode 100644 index 0000000..aebe230 --- /dev/null +++ b/scripts/inspect_glances_proxy.py @@ -0,0 +1,26 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('14631:'); +console.log(content.substring(idx, Math.min(content.length, idx + 1000))); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/glances_mod.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/glances_mod.js")) + +c.close() diff --git a/scripts/inspect_gotify_db.py b/scripts/inspect_gotify_db.py index 7bbdda8..188c42a 100644 --- a/scripts/inspect_gotify_db.py +++ b/scripts/inspect_gotify_db.py @@ -1,23 +1,23 @@ import paramiko -ssh = paramiko.SSHClient() -ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) -ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) -py_script = """sudo python3 -c ' -import sqlite3 -conn = sqlite3.connect("/mnt/swarm_shared/legacy_volumes/gotify/gotify.db") +py_script = """import sqlite3 +conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/gotify.db') cur = conn.cursor() -print("Applications:", cur.execute("SELECT id, name, description FROM applications").fetchall()) -print("Clients:", cur.execute("SELECT id, name FROM clients").fetchall()) -print("Messages count:", cur.execute("SELECT count(*) FROM messages").fetchone()[0]) -print("Recent messages:", cur.execute("SELECT id, title, message, date FROM messages ORDER BY id DESC LIMIT 5").fetchall()) -print("Users:", cur.execute("SELECT id, name, admin FROM users").fetchall()) -'""" +print('=== APPLICATIONS ===') +cols = [c[1] for c in cur.execute('PRAGMA table_info(applications)').fetchall()] +print('Cols:', cols) +for r in cur.execute('SELECT * FROM applications').fetchall(): + print(r) +""" -_, out, err = ssh.exec_command(py_script) -print(out.read().decode()) -print("ERR:", err.read().decode()) - -ssh.close() +stdin, stdout, stderr = c.exec_command("cat > /tmp/inspect_gotify.py && python3 /tmp/inspect_gotify.py") +stdin.write(py_script) +stdin.close() +print(stdout.read().decode()) +print(stderr.read().decode()) +c.close() diff --git a/scripts/inspect_grafana_proxy.py b/scripts/inspect_grafana_proxy.py new file mode 100644 index 0000000..677c21b --- /dev/null +++ b/scripts/inspect_grafana_proxy.py @@ -0,0 +1,27 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('81390:'); +console.log(content.substring(idx, Math.min(content.length, idx + 800))); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/check_81390.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/check_81390.js")) + +c.close() + diff --git a/scripts/inspect_grafana_widget_component.py b/scripts/inspect_grafana_widget_component.py new file mode 100644 index 0000000..d6c6ae2 --- /dev/null +++ b/scripts/inspect_grafana_widget_component.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def cat(path): + _, o, _ = c.exec_command(f'kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- cat {path}') + return o.read().decode().strip() + +print(cat("/app/src/widgets/grafana/component.jsx")) +c.close() + diff --git a/scripts/inspect_ha_widget_code.py b/scripts/inspect_ha_widget_code.py new file mode 100644 index 0000000..1ff37e4 --- /dev/null +++ b/scripts/inspect_ha_widget_code.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Pod:', pod) + +print('=== Searching for homeassistant widget file ===') +print(run(f'kubectl exec -n infrastructure {pod} -- find /app -name "*homeassistant*"')) + +c.close() diff --git a/scripts/inspect_ha_widget_files.py b/scripts/inspect_ha_widget_files.py new file mode 100644 index 0000000..cfd85a8 --- /dev/null +++ b/scripts/inspect_ha_widget_files.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Files in /app/src/widgets/homeassistant:') +print(run(f'kubectl exec -n infrastructure {pod} -- ls -la /app/src/widgets/homeassistant')) + +print('\nContent of component or widget:') +print(run(f'kubectl exec -n infrastructure {pod} -- find /app/src/widgets/homeassistant -type f -exec head -n 100 {{}} +')) + +c.close() diff --git a/scripts/inspect_heimdall_db.py b/scripts/inspect_heimdall_db.py new file mode 100644 index 0000000..f7f72b3 --- /dev/null +++ b/scripts/inspect_heimdall_db.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("df -h | grep -E 'nfs|docker'") +run("ls -la /mnt/nfs_shares/ 2>/dev/null || true") +run("ls -la /mnt/nfs_shares/heimdall/ 2>/dev/null || true") +run("ls -la /mnt/nfs_shares/heimdall/www/ 2>/dev/null || true") +c.close() + diff --git a/scripts/inspect_heimdall_schema.py b/scripts/inspect_heimdall_schema.py new file mode 100644 index 0000000..12ffdfb --- /dev/null +++ b/scripts/inspect_heimdall_schema.py @@ -0,0 +1,28 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("""python3 -c " +import sqlite3 +con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite') +cur = con.cursor() +cols = [d[1] for d in cur.execute('PRAGMA table_info(items);').fetchall()] +print('Items columns:', cols) +for r in cur.execute('SELECT * FROM items LIMIT 2;').fetchall(): + print(dict(zip(cols, r))) +" """) + +c.close() + diff --git a/scripts/inspect_heimdall_users.py b/scripts/inspect_heimdall_users.py new file mode 100644 index 0000000..faaeaab --- /dev/null +++ b/scripts/inspect_heimdall_users.py @@ -0,0 +1,28 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +run("""python3 -c " +import sqlite3 +con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite') +cur = con.cursor() +print('Users:', cur.execute('SELECT id, username, email FROM users;').fetchall()) +print('Items sample:') +for r in cur.execute('SELECT id, user_id, title, url, \"order\", pinned FROM items ORDER BY title, id LIMIT 10;').fetchall(): + print(r) +" """) + +c.close() + diff --git a/scripts/inspect_heimdall_watchtower.py b/scripts/inspect_heimdall_watchtower.py new file mode 100644 index 0000000..e343e93 --- /dev/null +++ b/scripts/inspect_heimdall_watchtower.py @@ -0,0 +1,26 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +run("docker stack ps heimdall") +run("docker stack ps watchtower") +run("docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'") +run("docker service inspect watchtower_watchtower --format '{{json .Spec.TaskTemplate.ContainerSpec}}'") +c.close() + diff --git a/scripts/inspect_homepage_backend.py b/scripts/inspect_homepage_backend.py new file mode 100644 index 0000000..2b53b9e --- /dev/null +++ b/scripts/inspect_homepage_backend.py @@ -0,0 +1,25 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +console.log(content.substring(69600, 71500)); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/print_ha.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/print_ha.js")) + +c.close() diff --git a/scripts/inspect_homepage_fs.py b/scripts/inspect_homepage_fs.py new file mode 100644 index 0000000..b023956 --- /dev/null +++ b/scripts/inspect_homepage_fs.py @@ -0,0 +1,32 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +pod = o.read().decode().strip() + +_, o, e = c.exec_command(f'kubectl exec -n infrastructure {pod} -- ls -la /app/public') +print("ls -la /app/public:\n", o.read().decode().strip()) + +# Test copying a file directly into the pod using cat +_, o, _ = c.exec_command(f'kubectl exec -i -n infrastructure {pod} -- sh -c "cat > /app/public/test.txt"') +# Wait, let's test if cat > /app/public/ works +stdin, stdout, stderr = c.exec_command(f'kubectl exec -i -n infrastructure {pod} -- sh -c "cat > /app/public/topology.html"') +with open('docs/snarfnet_homelab_topology.html', 'r', encoding='utf-8') as f: + stdin.write(f.read()) +stdin.close() +stdout.read() + +stdin, stdout, stderr = c.exec_command(f'kubectl exec -i -n infrastructure {pod} -- sh -c "cat > /app/public/topology.png"') +with open('docs/snarfnet_homelab_topology.png', 'rb') as f: + stdin.write(f.read()) +stdin.close() +stdout.read() + +_, o, e = c.exec_command(f'kubectl exec -n infrastructure {pod} -- ls -lh /app/public/topology*') +print("After streaming directly:\n", o.read().decode().strip()) +print("Stderr:\n", e.read().decode().strip()) + +c.close() diff --git a/scripts/inspect_homepage_info_widgets.py b/scripts/inspect_homepage_info_widgets.py new file mode 100644 index 0000000..01485a2 --- /dev/null +++ b/scripts/inspect_homepage_info_widgets.py @@ -0,0 +1,29 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print("=== /app/src/skeleton/widgets.yaml ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/skeleton/widgets.yaml")) + +print("\n=== /app/src/skeleton/kubernetes.yaml ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/skeleton/kubernetes.yaml")) + +print("\n=== /app/src/components/widgets/longhorn ===") +print(run(f"kubectl exec -n infrastructure {pod} -- ls -la /app/src/components/widgets/longhorn")) +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/components/widgets/longhorn/longhorn.jsx 2>/dev/null || cat /app/src/components/widgets/longhorn/*")) + +print("\n=== /app/src/components/widgets/kubernetes ===") +print(run(f"kubectl exec -n infrastructure {pod} -- ls -la /app/src/components/widgets/kubernetes")) +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/components/widgets/kubernetes/kubernetes.jsx")) + +c.close() + diff --git a/scripts/inspect_homepage_k8s.py b/scripts/inspect_homepage_k8s.py index 3719b51..61cf052 100644 --- a/scripts/inspect_homepage_k8s.py +++ b/scripts/inspect_homepage_k8s.py @@ -1,11 +1,19 @@ import paramiko -ssh = paramiko.SSHClient() -ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) -ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) -_, out, _ = ssh.exec_command('kubectl get all,ing,cm,pvc -n infrastructure | grep -i homepage') -print(out.read().decode()) +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out -ssh.close() +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print("Find in src/widgets or components:") +print(run(f"kubectl exec -n infrastructure {pod} -- find /app/src -maxdepth 3")) +print("\nKubernetes in /app:") +print(run(f"kubectl exec -n infrastructure {pod} -- find /app -name '*kubernetes*'")) + +c.close() diff --git a/scripts/inspect_homepage_k8s_code.py b/scripts/inspect_homepage_k8s_code.py new file mode 100644 index 0000000..cf12063 --- /dev/null +++ b/scripts/inspect_homepage_k8s_code.py @@ -0,0 +1,27 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/widgets/kubernetes.js', 'utf8'); +const idx = content.indexOf('getNodeMetrics'); +console.log(content.substring(Math.max(0, idx - 800), Math.min(content.length, idx + 800))); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/k8s_code.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/k8s_code.js")) + +c.close() + diff --git a/scripts/inspect_homepage_k8s_conf.py b/scripts/inspect_homepage_k8s_conf.py new file mode 100644 index 0000000..ad1881c --- /dev/null +++ b/scripts/inspect_homepage_k8s_conf.py @@ -0,0 +1,30 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/widgets/kubernetes.js', 'utf8'); +const idx = content.indexOf('kubernetes.yaml'); +console.log('Index of kubernetes.yaml:', idx); +if (idx !== -1) { + console.log(content.substring(Math.max(0, idx - 400), Math.min(content.length, idx + 400))); +} +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/k8s_conf.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/k8s_conf.js")) + +c.close() + diff --git a/scripts/inspect_homepage_k8s_longhorn.py b/scripts/inspect_homepage_k8s_longhorn.py new file mode 100644 index 0000000..bf7081b --- /dev/null +++ b/scripts/inspect_homepage_k8s_longhorn.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print("=== cat /app/.next/server/pages/api/widgets/longhorn.js ===") +print(run(f"kubectl exec -n infrastructure {pod} -- head -n 40 /app/.next/server/pages/api/widgets/longhorn.js")) + +print("\n=== cat /app/.next/server/pages/api/widgets/kubernetes.js ===") +print(run(f"kubectl exec -n infrastructure {pod} -- head -n 40 /app/.next/server/pages/api/widgets/kubernetes.js")) + +c.close() diff --git a/scripts/inspect_homepage_logs.py b/scripts/inspect_homepage_logs.py new file mode 100644 index 0000000..4b87ef4 --- /dev/null +++ b/scripts/inspect_homepage_logs.py @@ -0,0 +1,26 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") +if pod: + print(f"Homepage Pod: {pod}") + run(f"kubectl logs -n infrastructure {pod} --tail 60") + +c.close() + diff --git a/scripts/inspect_homepage_pod.py b/scripts/inspect_homepage_pod.py new file mode 100644 index 0000000..7fc9b8a --- /dev/null +++ b/scripts/inspect_homepage_pod.py @@ -0,0 +1,30 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + return res + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Pod name:', pod) + +print('=== services.yaml ===') +print(run(f'kubectl exec -n infrastructure {pod} -- cat /app/config/services.yaml')) + +print('=== widgets.yaml ===') +print(run(f'kubectl exec -n infrastructure {pod} -- cat /app/config/widgets.yaml')) + +print('=== settings.yaml ===') +print(run(f'kubectl exec -n infrastructure {pod} -- cat /app/config/settings.yaml')) + +print('=== Recent logs ===') +print(run(f'kubectl logs -n infrastructure {pod} --tail=30')) + +c.close() + diff --git a/scripts/inspect_icon_component.py b/scripts/inspect_icon_component.py new file mode 100644 index 0000000..3372789 --- /dev/null +++ b/scripts/inspect_icon_component.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") + +print("Files matching icon:") +print(run(f"kubectl exec -n infrastructure {pod} -- find /app/src -iname '*icon*'")) + +c.close() + diff --git a/scripts/inspect_k8s_multi.py b/scripts/inspect_k8s_multi.py new file mode 100644 index 0000000..2fd4d74 --- /dev/null +++ b/scripts/inspect_k8s_multi.py @@ -0,0 +1,27 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/widgets/kubernetes.js', 'utf8'); +const idx = content.indexOf('loadFromDefault'); +console.log(content.substring(Math.max(0, idx - 400), Math.min(content.length, idx + 400))); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/check_k8s_multi.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/check_k8s_multi.js")) + +c.close() + diff --git a/scripts/inspect_kuma_details.py b/scripts/inspect_kuma_details.py new file mode 100644 index 0000000..0ccf467 --- /dev/null +++ b/scripts/inspect_kuma_details.py @@ -0,0 +1,29 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +print("--- TABLES ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SHOW TABLES;" """)) + +print("\n--- TAG TABLE SCHEMA ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE tag; DESCRIBE monitor_tag;" """)) + +print("\n--- EXISTING TAGS ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM tag;" """)) + +print("\n--- ALL MONITORS ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, url, hostname, port, active FROM monitor ORDER BY id;" """)) + +print("\n--- EXISTING STATUS PAGES ---") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, slug, title, description, published FROM status_page;" """)) + +c.close() + diff --git a/scripts/inspect_kuma_node.py b/scripts/inspect_kuma_node.py new file mode 100644 index 0000000..761848a --- /dev/null +++ b/scripts/inspect_kuma_node.py @@ -0,0 +1,28 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data/") +run(f"""kubectl exec -n monitoring {pod} -- node -e ' +const Database = require("/app/node_modules/better-sqlite3"); +const db = new Database("/app/data/kuma.db"); +console.log("Monitors count:", db.prepare("SELECT count(*) as c FROM monitor").get().c); +console.log("Monitors:", db.prepare("SELECT id, name, type, url FROM monitor LIMIT 15").all()); +console.log("Notifications:", db.prepare("SELECT * FROM notification").all()); +' 2>/dev/null || true""") + +c.close() + diff --git a/scripts/inspect_kuma_proxy.py b/scripts/inspect_kuma_proxy.py new file mode 100644 index 0000000..d5e1da0 --- /dev/null +++ b/scripts/inspect_kuma_proxy.py @@ -0,0 +1,26 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('98938:'); +console.log(content.substring(idx, Math.min(content.length, idx + 800))); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/check_98938.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/check_98938.js")) + +c.close() diff --git a/scripts/inspect_kuma_schema.py b/scripts/inspect_kuma_schema.py new file mode 100644 index 0000000..c13dffb --- /dev/null +++ b/scripts/inspect_kuma_schema.py @@ -0,0 +1,25 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run_query(query): + cmd = f"""kubectl exec -n monitoring uptime-kuma-2-1756668961-6656d4d5c6-z5cz9 -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; {query}" """ + _, stdout, stderr = c.exec_command(cmd) + res = stdout.read().decode().strip() + if res: + print(res) + return res + +print("=== Monitor Table Schema ===") +run_query("DESCRIBE monitor;") +print("\n=== Monitor Notification Table Schema ===") +run_query("DESCRIBE monitor_notification;") +print("\n=== Notification Table Schema ===") +run_query("DESCRIBE notification;") + +c.close() + diff --git a/scripts/inspect_lh_conf.py b/scripts/inspect_lh_conf.py new file mode 100644 index 0000000..a9a79f7 --- /dev/null +++ b/scripts/inspect_lh_conf.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if err: + print("ERR:", err) + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print("cat /app/config/settings.yaml:") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/config/settings.yaml")) + +c.close() diff --git a/scripts/inspect_longhorn_data.py b/scripts/inspect_longhorn_data.py new file mode 100644 index 0000000..5c5e8bc --- /dev/null +++ b/scripts/inspect_longhorn_data.py @@ -0,0 +1,27 @@ +import json +import requests + +r = requests.get('http://140.44.4.11:31000/v1/nodes') +data = r.json() +print("Number of nodes:", len(data.get('data', []))) + +total_avail = 0 +total_max = 0 +total_sched = 0 + +for n in data.get('data', []): + name = n.get('name') + disks = n.get('disks', {}) + for d_name, d_info in disks.items(): + avail = d_info.get('storageAvailable', 0) + max_s = d_info.get('storageMaximum', 0) + sched = d_info.get('storageScheduled', 0) + total_avail += avail + total_max += max_s + total_sched += sched + +print(f"Total Max: {total_max / (1024**3):.2f} GiB ({total_max / (1024**4):.2f} TiB)") +print(f"Total Avail: {total_avail / (1024**3):.2f} GiB ({total_avail / (1024**4):.2f} TiB)") +used = total_max - total_avail +print(f"Total Used: {used / (1024**3):.2f} GiB ({used / (1024**4):.2f} TiB) ({used/total_max*100:.1f}%)") + diff --git a/scripts/inspect_npm_adv.py b/scripts/inspect_npm_adv.py new file mode 100644 index 0000000..57cc71c --- /dev/null +++ b/scripts/inspect_npm_adv.py @@ -0,0 +1,25 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.4.71", username="root", key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """import sqlite3 + +conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite') +cur = conn.cursor() +cur.execute('SELECT id, domain_names, advanced_config FROM proxy_host WHERE id IN (5, 22, 32, 56)') +for r in cur.fetchall(): + print(f'=== Host {r[0]}: {r[1]} ===') + print(r[2]) +conn.close() +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/check_adv.py") +stdin.write(py_script) +stdin.close() +stdout.read() + +stdin, stdout, stderr = c.exec_command("python3 /tmp/check_adv.py") +print(stdout.read().decode()) +c.close() diff --git a/scripts/inspect_npm_hosts.py b/scripts/inspect_npm_hosts.py index 55abc8d..46f3d77 100644 --- a/scripts/inspect_npm_hosts.py +++ b/scripts/inspect_npm_hosts.py @@ -1,18 +1,32 @@ import paramiko -ssh = paramiko.SSHClient() -ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) -ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect("140.44.4.71", username="root", key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) -py_script = """sudo python3 -c ' -import sqlite3 -conn = sqlite3.connect("/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite") +py_script = """import sqlite3, json + +conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite') cur = conn.cursor() -for row in cur.execute("select id,domain_names,forward_host,forward_port,enabled from proxy_host"): - print(row) -'""" -_, out, err = ssh.exec_command(py_script) -print(out.read().decode()) -print("ERR:", err.read().decode()) +cur.execute('SELECT id, domain_names, forward_host, forward_port, advanced_config FROM proxy_host WHERE is_deleted = 0') +rows = cur.fetchall() +for r in rows: + try: + domains = json.loads(r[1]) + except: + domains = r[1] + adv = r[4] or '' + has_auth = 'authentik' in adv.lower() or 'outpost' in adv.lower() + print('ID %2d: %s -> %s:%s (Authentik: %s)' % (r[0], domains, r[2], r[3], has_auth)) +conn.close() +""" -ssh.close() +stdin, stdout, stderr = c.exec_command("cat > /tmp/check_npm.py") +stdin.write(py_script) +stdin.close() +stdout.read() + +stdin, stdout, stderr = c.exec_command("python3 /tmp/check_npm.py") +print(stdout.read().decode()) +print(stderr.read().decode()) +c.close() diff --git a/scripts/inspect_peanut_widget.py b/scripts/inspect_peanut_widget.py new file mode 100644 index 0000000..56d7e9a --- /dev/null +++ b/scripts/inspect_peanut_widget.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def cat(path): + _, o, _ = c.exec_command(f'kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- cat {path}') + return o.read().decode().strip() + +print(cat("/app/src/widgets/peanut/component.jsx")) +c.close() + diff --git a/scripts/inspect_portainer_glances.py b/scripts/inspect_portainer_glances.py new file mode 100644 index 0000000..645bdb2 --- /dev/null +++ b/scripts/inspect_portainer_glances.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print("=== /app/src/widgets/portainer/component.jsx ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/widgets/portainer/component.jsx")) + +print("\n=== /app/src/widgets/glances/component.jsx ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/widgets/glances/component.jsx")) + +c.close() + diff --git a/scripts/inspect_portainer_header.py b/scripts/inspect_portainer_header.py new file mode 100644 index 0000000..9e67b48 --- /dev/null +++ b/scripts/inspect_portainer_header.py @@ -0,0 +1,26 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('41788'); +console.log(content.substring(Math.max(0, idx - 100), Math.min(content.length, idx + 800))); +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/search_portainer_header.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/search_portainer_header.js")) + +c.close() diff --git a/scripts/inspect_portainer_proxy.py b/scripts/inspect_portainer_proxy.py new file mode 100644 index 0000000..4506b1c --- /dev/null +++ b/scripts/inspect_portainer_proxy.py @@ -0,0 +1,28 @@ +import base64 +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +script = """ +const fs = require('fs'); +const content = fs.readFileSync('/app/.next/server/pages/api/services/proxy.js', 'utf8'); +const idx = content.indexOf('41788:'); +if (idx !== -1) { + console.log(content.substring(idx, Math.min(content.length, idx + 1000))); +} +""" + +b64 = base64.b64encode(script.encode('utf-8')).decode('ascii') +run(f"kubectl exec -n infrastructure {pod} -- sh -c 'echo {b64} | base64 -d > /tmp/portainer_mod.js'") +print(run(f"kubectl exec -n infrastructure {pod} -- node /tmp/portainer_mod.js")) + +c.close() diff --git a/scripts/inspect_prom_manifest.py b/scripts/inspect_prom_manifest.py new file mode 100644 index 0000000..9035d88 --- /dev/null +++ b/scripts/inspect_prom_manifest.py @@ -0,0 +1,26 @@ +import paramiko +import time +import yaml + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f: + text = f.read() + +docs = list(yaml.safe_load_all(text)) + +# Let's inspect docs +print("Docs count:", len(docs)) +for i, d in enumerate(docs): + print(i, d.get('kind'), d.get('metadata', {}).get('name')) + +c.close() + diff --git a/scripts/inspect_prom_widget.py b/scripts/inspect_prom_widget.py new file mode 100644 index 0000000..dd9ef3e --- /dev/null +++ b/scripts/inspect_prom_widget.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print("=== /app/src/widgets/prometheus/component.jsx ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/widgets/prometheus/component.jsx")) + +print("\n=== /app/src/widgets/prometheusmetric/component.jsx ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/widgets/prometheusmetric/component.jsx")) + +c.close() + diff --git a/scripts/inspect_routes.py b/scripts/inspect_routes.py new file mode 100644 index 0000000..3549dc5 --- /dev/null +++ b/scripts/inspect_routes.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app/src -name \"*proxy*\" -o -name \"route.js\" -o -name \"route.ts\"")) +c.close() + diff --git a/scripts/inspect_service_widgets.py b/scripts/inspect_service_widgets.py new file mode 100644 index 0000000..ea36ab1 --- /dev/null +++ b/scripts/inspect_service_widgets.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +for w in ['grafana', 'uptimekuma', 'gitea', 'argocd']: + print(f"=== {w} component.jsx ===") + print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/widgets/{w}/component.jsx")) + +c.close() + diff --git a/scripts/inspect_skeleton_settings.py b/scripts/inspect_skeleton_settings.py new file mode 100644 index 0000000..8c6fab0 --- /dev/null +++ b/scripts/inspect_skeleton_settings.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') + +print("=== /app/src/skeleton/settings.yaml ===") +print(run(f"kubectl exec -n infrastructure {pod} -- cat /app/src/skeleton/settings.yaml")) + +c.close() + diff --git a/scripts/inspect_uptime_kuma.py b/scripts/inspect_uptime_kuma.py new file mode 100644 index 0000000..a5c6c69 --- /dev/null +++ b/scripts/inspect_uptime_kuma.py @@ -0,0 +1,30 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +run("kubectl get pods,svc,ingress -n monitoring -l app.kubernetes.io/name=uptime-kuma") +run("kubectl get svc -n monitoring") +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +if pod: + print(f"Uptime Kuma Pod: {pod}") + run(f"kubectl exec -n monitoring {pod} -- sqlite3 /app/data/kuma.db 'SELECT count(*) FROM monitor;' 2>/dev/null || echo 'sqlite3 check inside pod'") + run(f"kubectl exec -n monitoring {pod} -- sqlite3 /app/data/kuma.db 'SELECT id, name, type, url FROM monitor LIMIT 10;' 2>/dev/null || true") + run(f"kubectl exec -n monitoring {pod} -- sqlite3 /app/data/kuma.db 'SELECT id, name, type FROM notification;' 2>/dev/null || true") + +c.close() + diff --git a/scripts/inspect_use_widget_api.py b/scripts/inspect_use_widget_api.py new file mode 100644 index 0000000..2389ac5 --- /dev/null +++ b/scripts/inspect_use_widget_api.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def cat(path): + _, o, _ = c.exec_command(f'kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- cat {path}') + return o.read().decode().strip() + +print(cat("/app/src/utils/proxy/use-widget-api.js")) +c.close() + diff --git a/scripts/inspect_widget_endpoints.py b/scripts/inspect_widget_endpoints.py new file mode 100644 index 0000000..3a304fa --- /dev/null +++ b/scripts/inspect_widget_endpoints.py @@ -0,0 +1,14 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print("Files:") +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- find /app/src/widgets -type d -maxdepth 2")) +c.close() + diff --git a/scripts/inspect_widget_files.py b/scripts/inspect_widget_files.py new file mode 100644 index 0000000..69bdd9c --- /dev/null +++ b/scripts/inspect_widget_files.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print("uptimekuma:", run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- ls -la /app/src/widgets/uptimekuma")) +print("homeassistant:", run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- ls -la /app/src/widgets/homeassistant")) +print("gitea:", run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- ls -la /app/src/widgets/gitea")) + +c.close() + diff --git a/scripts/inspect_widget_proxies.py b/scripts/inspect_widget_proxies.py new file mode 100644 index 0000000..f96cb4b --- /dev/null +++ b/scripts/inspect_widget_proxies.py @@ -0,0 +1,21 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def cat(path): + _, o, _ = c.exec_command(f'kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- cat {path}') + return o.read().decode().strip() + +print("--- gitea proxy ---") +print(cat("/app/src/widgets/gitea/proxy.js")) + +print("--- uptimekuma proxy ---") +print(cat("/app/src/widgets/uptimekuma/proxy.js")) + +print("--- homeassistant proxy ---") +print(cat("/app/src/widgets/homeassistant/proxy.js")) + +c.close() + diff --git a/scripts/list_app.py b/scripts/list_app.py new file mode 100644 index 0000000..583ae3f --- /dev/null +++ b/scripts/list_app.py @@ -0,0 +1,17 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('ls -la /app:') +print(run(f'kubectl exec -n infrastructure {pod} -- ls -la /app')) +print('find /app/.next -name "*homeassistant*":') +print(run(f'kubectl exec -n infrastructure {pod} -- find /app/.next -name "*homeassistant*"')) + +c.close() diff --git a/scripts/list_homepage_widgets.py b/scripts/list_homepage_widgets.py new file mode 100644 index 0000000..8a65135 --- /dev/null +++ b/scripts/list_homepage_widgets.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print("Pod:", pod) + +widgets = run(f"kubectl exec -n infrastructure {pod} -- ls /app/src/widgets") +print("Widgets in /app/src/widgets:") +print(widgets) + +c.close() + diff --git a/scripts/list_st_entities.py b/scripts/list_st_entities.py new file mode 100644 index 0000000..60a70be --- /dev/null +++ b/scripts/list_st_entities.py @@ -0,0 +1,23 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +ha_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4" + +cmd = f"""curl -s -H "Authorization: Bearer {ha_token}" -H "Content-Type: application/json" http://14.4.4.20:8123/api/states""" +_, stdout, stderr = c.exec_command(cmd) +raw = stdout.read().decode() + +states = json.loads(raw) +st_entities = [s for s in states if s['entity_id'].startswith('sensor.st_00158555')] +print(f"Total Weather Station ST-00158555 entities: {len(st_entities)}") +for s in st_entities: + print(f" {s['entity_id']} = {s['state']} {s.get('attributes', {}).get('unit_of_measurement', '')} ({s.get('attributes', {}).get('friendly_name')})") + +c.close() + diff --git a/scripts/ls_peanut.py b/scripts/ls_peanut.py new file mode 100644 index 0000000..b767546 --- /dev/null +++ b/scripts/ls_peanut.py @@ -0,0 +1,13 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print(run("kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- ls -la /app/src/widgets/peanut")) +c.close() + diff --git a/scripts/patch_aerodrome_prom.py b/scripts/patch_aerodrome_prom.py new file mode 100644 index 0000000..7e1a047 --- /dev/null +++ b/scripts/patch_aerodrome_prom.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +patch = '{"spec":{"type":"NodePort","ports":[{"name":"web","port":9090,"targetPort":9090,"nodePort":30090}]}}' +_, o, e = c.exec_command(f"kubectl patch svc prometheus -n default --type merge -p '{patch}'") +print(o.read().decode()) +print(e.read().decode()) + +_, o2, _ = c.exec_command("kubectl get svc prometheus -n default") +print(o2.read().decode()) + +c.close() + diff --git a/scripts/populate_group.py b/scripts/populate_group.py new file mode 100644 index 0000000..27a813e --- /dev/null +++ b/scripts/populate_group.py @@ -0,0 +1,26 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +sql = """ +INSERT INTO `group` (id, name, public, active, weight, status_page_id) +VALUES (1, 'Core Infrastructure', 1, 1, 1000, 1) +ON DUPLICATE KEY UPDATE name=VALUES(name); + +INSERT INTO monitor_group (monitor_id, group_id, weight) +SELECT id, 1, 1000 FROM monitor WHERE active=1; +""" + +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e \"{sql}\"")) +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT count(id) FROM monitor_group;'")) + +c.close() + diff --git a/scripts/populate_status_page.py b/scripts/populate_status_page.py new file mode 100644 index 0000000..f86f9f8 --- /dev/null +++ b/scripts/populate_status_page.py @@ -0,0 +1,30 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +sql = """ +INSERT INTO status_page (id, slug, title, description, icon, theme, published, search_engine_index) +VALUES (1, 'default', 'SnarfNet Infrastructure', 'Overall health of SnarfNet endpoints and services', '/icon.svg', 'auto', 1, 1) +ON DUPLICATE KEY UPDATE title=VALUES(title); + +INSERT INTO `group` (id, name, public, active, weight, status_page_id) +VALUES (1, 'Core Infrastructure', 1, 1, 1000, 1) +ON DUPLICATE KEY UPDATE name=VALUES(name); + +INSERT IGNORE INTO monitor_group (monitor_id, group_id, weight) +SELECT id, 1, 1000 FROM monitor WHERE active=1; +""" + +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e \"{sql}\"")) +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT count(*) FROM monitor_group;'")) + +c.close() + diff --git a/scripts/populate_uptime_kuma.py b/scripts/populate_uptime_kuma.py new file mode 100644 index 0000000..cba2677 --- /dev/null +++ b/scripts/populate_uptime_kuma.py @@ -0,0 +1,170 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run_cmd(cmd): + print(f"\n>>> {cmd}") + _, stdout, stderr = c.exec_command(cmd) + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + return out + +pod = run_cmd("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +# Remote python script to populate MariaDB +setup_script = ''' +import json +import subprocess + +def query_db(sql): + cmd = ["mariadb", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-e", sql] + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode != 0: + print("DB ERROR:", res.stderr) + return res.stdout + +# 1. Ensure Gotify Notification Exists +notif_config = { + "name": "Gotify Infrastructure Alerts", + "type": "gotify", + "isDefault": True, + "gotifyServerUrl": "http://140.44.4.70:8090", + "gotifyApplicationToken": "A7zBx9InFrAaLeRtStOk", + "gotifyPriority": 8 +} +notif_json = json.dumps(notif_config) + +check_notif = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';") +if "id" not in check_notif or not check_notif.split(): + print("Creating Gotify Notification in Uptime Kuma...") + escaped_json = notif_json.replace("'", "''") + query_db(f"INSERT INTO notification (name, active, user_id, is_default, config) VALUES ('Gotify Infrastructure Alerts', 1, 1, 1, '{escaped_json}');") + +notif_id_out = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';") +notif_id = [line.strip() for line in notif_id_out.splitlines() if line.strip().isdigit()][0] +print(f"Gotify Notification ID: {notif_id}") + +# 2. Define Monitors List (DNS + Direct VIP/IP) +monitors = [ + # --- DNS INGRESS MONITORS --- + {"name": "Gotify Hub (DNS)", "type": "http", "url": "https://gotify.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Heimdall Portal (DNS)", "type": "http", "url": "https://portal.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "NPM Admin Console (DNS)", "type": "http", "url": "https://nginx.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Authentik SSO (DNS)", "type": "http", "url": "https://authentik.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-399"]'}, + {"name": "Rancher Management (DNS)", "type": "http", "url": "https://aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Harvester Console (DNS)", "type": "http", "url": "https://harvester.spacedock.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Gitea DevOps (DNS)", "type": "http", "url": "https://git.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "ArgoCD GitOps (DNS)", "type": "http", "url": "https://argo.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "AWX Automation (DNS)", "type": "http", "url": "https://awx.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Semaphore Ansible (DNS)", "type": "http", "url": "https://semaphore.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Jenkins CI/CD (DNS)", "type": "http", "url": "https://jenkins.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Zabbix Web UI (DNS)", "type": "http", "url": "https://zabbix.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Grafana Dashboards (DNS)", "type": "http", "url": "https://grafana.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Longhorn Aerodrome (DNS)", "type": "http", "url": "https://longhorn.aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Longhorn Spaceport (DNS)", "type": "http", "url": "https://longhorn.spaceport.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Home Assistant (DNS)", "type": "http", "url": "https://snarfhome.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "PeaNUT UPS Portal (DNS)", "type": "http", "url": "https://peanut.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Open WebUI AI (DNS)", "type": "http", "url": "https://ai.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "SnarfNAS DSM (DNS)", "type": "http", "url": "https://snarfnas.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "SN-NAS DSM (DNS)", "type": "http", "url": "https://sn-nas.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Homepage (DNS)", "type": "http", "url": "https://homepage.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "MeshCentral (DNS)", "type": "http", "url": "https://mesh.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Homebox (DNS)", "type": "http", "url": "https://homebox.snarfnet.net", "interval": 60, "ignore_tls": 0}, + + # --- DIRECT VIP & IP MONITORS --- + {"name": "Swarm VIP - Gotify (VIP)", "type": "http", "url": "http://140.44.4.70:8090", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm VIP - Heimdall (VIP)", "type": "http", "url": "http://140.44.4.70:85", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm VIP - Homebox (VIP)", "type": "http", "url": "http://140.44.4.70:3100", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm VIP - PeaNUT (VIP)", "type": "http", "url": "http://140.44.4.70:8050", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm Host - NPM (VIP)", "type": "http", "url": "http://140.44.4.74:81", "interval": 30, "ignore_tls": 0}, + {"name": "Aerodrome VIP - Authentik (VIP)", "type": "http", "url": "https://140.44.4.29:444", "interval": 30, "ignore_tls": 1, "status_codes": '["200-399"]'}, + {"name": "Aerodrome VIP - AWX (VIP)", "type": "http", "url": "https://140.44.4.30:443", "interval": 30, "ignore_tls": 1}, + {"name": "Aerodrome VIP - Grafana (VIP)", "type": "http", "url": "http://140.44.4.32:88", "interval": 30, "ignore_tls": 0}, + {"name": "Aerodrome VIP - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.36:80", "interval": 30, "ignore_tls": 0}, + {"name": "Aerodrome VIP - Traefik (VIP)", "type": "http", "url": "https://140.44.4.34:443", "interval": 30, "ignore_tls": 1}, + {"name": "Spaceport VIP - Zabbix Web (VIP)", "type": "http", "url": "http://140.44.4.57:8888", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Semaphore (VIP)", "type": "http", "url": "http://140.44.4.58:100", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Jenkins (VIP)", "type": "http", "url": "http://140.44.4.59:101", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Homepage (VIP)", "type": "http", "url": "http://140.44.4.60:8040", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Gitea (VIP)", "type": "http", "url": "http://140.44.4.62:90", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport Node - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.41:80", "interval": 30, "ignore_tls": 0}, + {"name": "Home Assistant Core (IP)", "type": "http", "url": "http://14.4.4.20:8123", "interval": 30, "ignore_tls": 0}, + {"name": "AI Hailo Engine (IP)", "type": "http", "url": "http://140.44.4.7:8000/docs", "interval": 30, "ignore_tls": 0}, + {"name": "AI Open WebUI (IP)", "type": "http", "url": "http://140.44.4.8:8080", "interval": 30, "ignore_tls": 0}, + {"name": "Harvester HCI Console (VIP)", "type": "http", "url": "https://140.44.4.90:443", "interval": 30, "ignore_tls": 1}, + {"name": "Synology NAS 1 DSM (IP)", "type": "http", "url": "https://14.10.10.71:5001", "interval": 30, "ignore_tls": 1}, + {"name": "SnarfNAS DSM (IP)", "type": "http", "url": "https://14.10.10.72:5001", "interval": 30, "ignore_tls": 1}, + {"name": "NUT Master Daemon (TCP)", "type": "port", "hostname": "14.4.4.44", "port": 3493, "interval": 30}, +] + +# Fetch existing monitors +existing_out = query_db("SELECT id, name FROM monitor;") +existing_names = set() +for line in existing_out.splitlines()[1:]: + parts = line.split("\\t") + if len(parts) >= 2: + existing_names.add(parts[1].strip()) + +added_count = 0 +for m in monitors: + name = m["name"] + if name in existing_names: + print(f"Skipping existing monitor: {name}") + continue + + m_type = m["type"] + interval = m.get("interval", 60) + ignore_tls = m.get("ignore_tls", 0) + status_codes = m.get("status_codes", '["200-299"]') + + if m_type == "http": + url = m["url"] + sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, url, ignore_tls, accepted_statuscodes_json) + VALUES ('{name}', 'http', 1, 1, {interval}, '{url}', {ignore_tls}, '{status_codes}');""" + elif m_type == "port": + host = m["hostname"] + p = m["port"] + sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, hostname, port) + VALUES ('{name}', 'port', 1, 1, {interval}, '{host}', {p});""" + + query_db(sql) + added_count += 1 + print(f"Added monitor: {name}") + +# Link all monitors to Gotify notification in monitor_notification +all_monitors_out = query_db("SELECT id FROM monitor;") +for line in all_monitors_out.splitlines()[1:]: + line = line.strip() + if line.isdigit(): + m_id = line + check_link = query_db(f"SELECT id FROM monitor_notification WHERE monitor_id = {m_id} AND notification_id = {notif_id};") + if "id" not in check_link or not check_link.split(): + query_db(f"INSERT INTO monitor_notification (monitor_id, notification_id) VALUES ({m_id}, {notif_id});") + +print(f"Successfully processed all monitors! Total new monitors added: {added_count}") +''' + +# Deliver script via stdin into /tmp/populate_kuma.py on the node +stdin, _, _ = c.exec_command("cat > /tmp/populate_kuma.py") +stdin.write(setup_script) +stdin.close() + +run_cmd(f"kubectl cp /tmp/populate_kuma.py monitoring/{pod}:/tmp/populate_kuma.py") +run_cmd(f"kubectl exec -n monitoring {pod} -- python3 /tmp/populate_kuma.py") + +# Restart Uptime Kuma to load new monitors +print("\n=== Restarting Uptime Kuma Deployment ===") +run_cmd("kubectl rollout restart deployment/uptime-kuma-2-1756668961 -n monitoring") +run_cmd("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=90s") + +c.close() +print("\n=== Uptime Kuma Population Complete ===") diff --git a/scripts/query_nas_from_swarm.py b/scripts/query_nas_from_swarm.py new file mode 100644 index 0000000..15e98ce --- /dev/null +++ b/scripts/query_nas_from_swarm.py @@ -0,0 +1,52 @@ +import paramiko, json + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def query_nas(ip, name): + print(f"\n=== Querying {name} ({ip}) from Swm01 ===") + for user in ['snarf', 'admin']: + login_cmd = f"curl -k -s 'https://{ip}:5001/webapi/auth.cgi?api=SYNO.API.Auth&version=3&method=login&account={user}&passwd=Katzir476!&session=Storage&format=sid'" + _, o, _ = c.exec_command(login_cmd) + resp_str = o.read().decode().strip() + try: + res = json.loads(resp_str) + if res.get('success'): + sid = res['data']['sid'] + print(f"Logged in as {user}!") + st_cmd = f"curl -k -s 'https://{ip}:5001/webapi/entry.cgi?api=SYNO.Storage.CGI.Storage&version=1&method=load_info&_sid={sid}'" + _, o2, _ = c.exec_command(st_cmd) + st_data = json.loads(o2.read().decode()) + data = st_data.get('data', {}) + vols = data.get('volumes', []) + disks = data.get('disks', []) + print(f"Disks count: {len(disks)}") + tot_raw = 0 + for d in disks: + sz = int(d.get('size_total', 0)) / (1024**4) + tot_raw += sz + print(f" Drive {d.get('name')}: {d.get('model', 'Unknown')}, Size: {sz:.2f} TiB, Status: {d.get('status')}") + print(f"Total Raw Drive Capacity: {tot_raw:.2f} TiB") + + print(f"Volumes count: {len(vols)}") + tot_vol = 0 + tot_used = 0 + for v in vols: + v_tot = int(v.get('size_total', 0)) / (1024**4) + v_usd = int(v.get('size_used', 0)) / (1024**4) + tot_vol += v_tot + tot_used += v_usd + print(f" Volume {v.get('name')}: Usable: {v_tot:.2f} TiB, Used: {v_usd:.2f} TiB, FS: {v.get('fs_type')}") + print(f"Total Usable Volume: {tot_vol:.2f} TiB (Used: {tot_used:.2f} TiB)") + break + else: + print(f"Login failed as {user}: {res}") + except Exception as e: + print(f"Error parse: {e}, raw: {resp_str[:150]}") + +query_nas('14.10.10.71', 'SN-NAS (RS810+)') +query_nas('14.10.10.72', 'SnarfNAS (DS416play)') + +c.close() + diff --git a/scripts/query_synology_dsm_storage.py b/scripts/query_synology_dsm_storage.py new file mode 100644 index 0000000..937b3ed --- /dev/null +++ b/scripts/query_synology_dsm_storage.py @@ -0,0 +1,62 @@ +import urllib.request, urllib.parse, ssl, json + +ctx = ssl._create_unverified_context() + +for ip, nas_name in [('14.10.10.71', 'SN-NAS (RS810+)'), ('14.10.10.72', 'SnarfNAS (DS416play)')]: + print(f"\n=== Querying {nas_name} ({ip}) ===") + logged_in = False + for user in ['snarf', 'admin']: + params = urllib.parse.urlencode({ + 'api': 'SYNO.API.Auth', + 'version': '3', + 'method': 'login', + 'account': user, + 'passwd': 'Katzir476!', + 'session': 'Storage', + 'format': 'sid' + }) + url = f'https://{ip}:5001/webapi/auth.cgi?{params}' + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + try: + with urllib.request.urlopen(req, context=ctx, timeout=4) as r: + res = json.loads(r.read().decode()) + if res.get('success'): + sid = res['data']['sid'] + print(f"Logged in successfully as {user}!") + logged_in = True + + st_url = f'https://{ip}:5001/webapi/entry.cgi?api=SYNO.Storage.CGI.Storage&version=1&method=load_info&_sid={sid}' + req2 = urllib.request.Request(st_url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req2, context=ctx, timeout=6) as r2: + st_data = json.loads(r2.read().decode()) + data = st_data.get('data', {}) + vols = data.get('volumes', []) + disks = data.get('disks', []) + storage_pools = data.get('storagePools', []) + + print(f"Disks count: {len(disks)}") + total_raw_tib = 0 + for d in disks: + sz = int(d.get('size_total', 0)) / (1024**4) + total_raw_tib += sz + print(f" Drive {d.get('name') or d.get('id')}: {d.get('model', 'Unknown')}, Size: {sz:.2f} TiB, Status: {d.get('status')}") + print(f"Total Raw Drive Capacity: {total_raw_tib:.2f} TiB") + + print(f"Volumes count: {len(vols)}") + total_usable_tib = 0 + total_used_tib = 0 + for v in vols: + v_tot = int(v.get('size_total', 0)) / (1024**4) + v_usd = int(v.get('size_used', 0)) / (1024**4) + total_usable_tib += v_tot + total_used_tib += v_usd + print(f" Volume {v.get('name', 'vol')}: Usable: {v_tot:.2f} TiB, Used: {v_usd:.2f} TiB, FS: {v.get('fs_type')}") + print(f"Total Usable Storage: {total_usable_tib:.2f} TiB (Used: {total_used_tib:.2f} TiB)") + break + else: + print(f"Login failed for {user}: {res}") + except Exception as e: + print(f"Error connecting as {user}: {e}") + if not logged_in: + print(f"Could not log into {ip}") + diff --git a/scripts/read_widget_components.py b/scripts/read_widget_components.py new file mode 100644 index 0000000..796458b --- /dev/null +++ b/scripts/read_widget_components.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def cat(path): + _, o, _ = c.exec_command(f'kubectl exec deployment/homepage-2-1744068158 -n infrastructure -- cat {path}') + return o.read().decode().strip() + +print("--- homeassistant ---") +print(cat("/app/src/widgets/homeassistant/component.jsx")) +print("--- uptimekuma ---") +print(cat("/app/src/widgets/uptimekuma/component.jsx")) +print("--- gitea ---") +print(cat("/app/src/widgets/gitea/component.jsx")) + +c.close() + diff --git a/scripts/recover_uptime_kuma.py b/scripts/recover_uptime_kuma.py new file mode 100644 index 0000000..e076b04 --- /dev/null +++ b/scripts/recover_uptime_kuma.py @@ -0,0 +1,176 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run_cmd(cmd): + print(f"\n>>> {cmd}") + _, stdout, stderr = c.exec_command(cmd) + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + return out + +pod = run_cmd("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print(f"Targeting Uptime Kuma Pod: {pod}") + +setup_script = '''import json +import subprocess + +def query_db(sql): + cmd = ["mariadb", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-e", sql] + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode != 0: + print("DB ERROR:", res.stderr) + return res.stdout + +# 1. Take a full MariaDB dump before recovery +subprocess.run(["mysqldump", "--socket=/app/data/run/mariadb.sock", "-u", "root", "kuma", "-r", "/app/data/pre_recovery_dump.sql"]) +print("Created pre-recovery backup at /app/data/pre_recovery_dump.sql") + +# 2. Ensure Gotify Notification Exists +notif_config = { + "name": "Gotify Infrastructure Alerts", + "type": "gotify", + "isDefault": True, + "gotifyServerUrl": "http://140.44.4.70:8090", + "gotifyApplicationToken": "A7zBx9InFrAaLeRtStOk", + "gotifyPriority": 8 +} +notif_json = json.dumps(notif_config) + +check_notif = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';") +if "id" not in check_notif or not check_notif.split(): + print("Creating Gotify Notification in Uptime Kuma...") + escaped_json = notif_json.replace("'", "''") + query_db(f"INSERT INTO notification (name, active, user_id, is_default, config) VALUES ('Gotify Infrastructure Alerts', 1, 1, 1, '{escaped_json}');") + +notif_id_out = query_db("SELECT id FROM notification WHERE name = 'Gotify Infrastructure Alerts';") +notif_id = [line.strip() for line in notif_id_out.splitlines() if line.strip().isdigit()][0] +print(f"Gotify Notification ID: {notif_id}") + +# 3. Clean any broken previous monitors +query_db("DELETE FROM monitor WHERE name IN ('Harvester Console (DNS)', 'Jenkins CI/CD (DNS)', 'NUT Master Daemon (TCP)');") + +# 4. Define Complete Fleet Monitors List (DNS + Direct VIP/IP) +monitors = [ + # --- DNS INGRESS MONITORS --- + {"name": "Gotify Hub (DNS)", "type": "http", "url": "https://gotify.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Heimdall Portal (DNS)", "type": "http", "url": "https://portal.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "NPM Admin Console (DNS)", "type": "http", "url": "https://nginx.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Authentik SSO (DNS)", "type": "http", "url": "https://authentik.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-399"]'}, + {"name": "Rancher Management (DNS)", "type": "http", "url": "https://aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Harvester Console (DNS)", "type": "http", "url": "https://harvester.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Gitea DevOps (DNS)", "type": "http", "url": "https://git.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "ArgoCD GitOps (DNS)", "type": "http", "url": "https://argo.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "AWX Automation (DNS)", "type": "http", "url": "https://awx.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Semaphore Ansible (DNS)", "type": "http", "url": "https://semaphore.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Jenkins CI/CD (DNS)", "type": "http", "url": "https://jenkins.snarfnet.net", "interval": 60, "ignore_tls": 0, "status_codes": '["200-403"]'}, + {"name": "Zabbix Web UI (DNS)", "type": "http", "url": "https://zabbix.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Grafana Dashboards (DNS)", "type": "http", "url": "https://grafana.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Longhorn Aerodrome (DNS)", "type": "http", "url": "https://longhorn.aerodrome.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Longhorn Spaceport (DNS)", "type": "http", "url": "https://longhorn.spaceport.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Home Assistant (DNS)", "type": "http", "url": "https://snarfhome.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "PeaNUT UPS Portal (DNS)", "type": "http", "url": "https://peanut.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "Open WebUI AI (DNS)", "type": "http", "url": "https://ai.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "SnarfNAS DSM (DNS)", "type": "http", "url": "https://snarfnas.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "SN-NAS DSM (DNS)", "type": "http", "url": "https://sn-nas.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Homepage (DNS)", "type": "http", "url": "https://homepage.snarfnet.net", "interval": 60, "ignore_tls": 0}, + {"name": "MeshCentral (DNS)", "type": "http", "url": "https://mesh.snarfnet.net", "interval": 60, "ignore_tls": 1}, + {"name": "Homebox (DNS)", "type": "http", "url": "https://homebox.snarfnet.net", "interval": 60, "ignore_tls": 0}, + + # --- DIRECT VIP & IP MONITORS --- + {"name": "Swarm VIP - Gotify (VIP)", "type": "http", "url": "http://140.44.4.70:8090", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm VIP - Heimdall (VIP)", "type": "http", "url": "http://140.44.4.70:85", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm VIP - Homebox (VIP)", "type": "http", "url": "http://140.44.4.70:3100", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm VIP - PeaNUT (VIP)", "type": "http", "url": "http://140.44.4.70:8050", "interval": 30, "ignore_tls": 0}, + {"name": "Swarm Host - NPM (VIP)", "type": "http", "url": "http://140.44.4.74:81", "interval": 30, "ignore_tls": 0}, + {"name": "Aerodrome VIP - Authentik (VIP)", "type": "http", "url": "https://140.44.4.29:444", "interval": 30, "ignore_tls": 1, "status_codes": '["200-399"]'}, + {"name": "Aerodrome VIP - AWX (VIP)", "type": "http", "url": "https://140.44.4.30:443", "interval": 30, "ignore_tls": 1}, + {"name": "Aerodrome VIP - Grafana (VIP)", "type": "http", "url": "http://140.44.4.32:88", "interval": 30, "ignore_tls": 0}, + {"name": "Aerodrome VIP - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.36:80", "interval": 30, "ignore_tls": 0}, + {"name": "Aerodrome VIP - Traefik (VIP)", "type": "http", "url": "https://140.44.4.34:443", "interval": 30, "ignore_tls": 1}, + {"name": "Spaceport VIP - Zabbix Web (VIP)", "type": "http", "url": "http://140.44.4.57:8888", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Semaphore (VIP)", "type": "http", "url": "http://140.44.4.58:100", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Jenkins (VIP)", "type": "http", "url": "http://140.44.4.59:101", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Homepage (VIP)", "type": "http", "url": "http://140.44.4.60:8040", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport VIP - Gitea (VIP)", "type": "http", "url": "http://140.44.4.62:90", "interval": 30, "ignore_tls": 0}, + {"name": "Spaceport Node - Longhorn (VIP)", "type": "http", "url": "http://140.44.4.41:80", "interval": 30, "ignore_tls": 0}, + {"name": "Home Assistant Core (IP)", "type": "http", "url": "http://14.4.4.20:8123", "interval": 30, "ignore_tls": 0}, + {"name": "AI Hailo Engine (IP)", "type": "http", "url": "http://140.44.4.7:8000/docs", "interval": 30, "ignore_tls": 0}, + {"name": "AI Open WebUI (IP)", "type": "http", "url": "http://140.44.4.8:8080", "interval": 30, "ignore_tls": 0}, + {"name": "Harvester HCI Console (VIP)", "type": "http", "url": "https://140.44.4.90:443", "interval": 30, "ignore_tls": 1}, + {"name": "Synology NAS 1 DSM (IP)", "type": "http", "url": "https://14.10.10.71:5001", "interval": 30, "ignore_tls": 1}, + {"name": "SnarfNAS DSM (IP)", "type": "http", "url": "https://14.10.10.72:5001", "interval": 30, "ignore_tls": 1}, + {"name": "PeaNUT UPS Daemon (VIP)", "type": "http", "url": "http://140.44.4.70:8050/api/v1/devices", "interval": 30, "ignore_tls": 0}, +] + +# Fetch existing monitors to prevent duplicates +existing_out = query_db("SELECT id, name FROM monitor;") +existing_names = set() +for line in existing_out.splitlines()[1:]: + parts = line.split("\\t") + if len(parts) >= 2: + existing_names.add(parts[1].strip()) + +added_count = 0 +for m in monitors: + name = m["name"] + if name in existing_names: + print(f"Skipping existing monitor: {name}") + continue + + m_type = m["type"] + interval = m.get("interval", 60) + ignore_tls = m.get("ignore_tls", 0) + status_codes = m.get("status_codes", '["200-299"]') + + if m_type == "http": + url = m["url"] + sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, url, ignore_tls, accepted_statuscodes_json) + VALUES ('{name}', 'http', 1, 1, {interval}, '{url}', {ignore_tls}, '{status_codes}');""" + elif m_type == "port": + host = m["hostname"] + p = m["port"] + sql = f"""INSERT INTO monitor (name, type, active, user_id, `interval`, hostname, port) + VALUES ('{name}', 'port', 1, 1, {interval}, '{host}', {p});""" + + query_db(sql) + added_count += 1 + print(f"Added monitor: {name}") + +# Link all monitors to Gotify notification +all_monitors_out = query_db("SELECT id FROM monitor;") +for line in all_monitors_out.splitlines()[1:]: + line = line.strip() + if line.isdigit(): + m_id = line + check_link = query_db(f"SELECT id FROM monitor_notification WHERE monitor_id = {m_id} AND notification_id = {notif_id};") + if "id" not in check_link or not check_link.split(): + query_db(f"INSERT INTO monitor_notification (monitor_id, notification_id) VALUES ({m_id}, {notif_id});") + +total_count = query_db("SELECT count(*) FROM monitor;") +print(f"\\nRecovery Complete! Added {added_count} monitors. Total active monitors in DB: {total_count.strip()}") +''' + +stdin, _, _ = c.exec_command("cat > /tmp/recover_kuma.py") +stdin.write(setup_script) +stdin.close() + +run_cmd(f"kubectl cp /tmp/recover_kuma.py monitoring/{pod}:/tmp/recover_kuma.py") +run_cmd(f"kubectl exec -n monitoring {pod} -- python3 /tmp/recover_kuma.py") + +print("\n=== Restarting Uptime Kuma Deployment ===") +run_cmd("kubectl rollout restart deployment/uptime-kuma-2-1756668961 -n monitoring") +run_cmd("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=90s") + +c.close() +print("\n=== Recovery Successfully Finished ===") + diff --git a/scripts/remediate_heimdall.py b/scripts/remediate_heimdall.py new file mode 100644 index 0000000..afacf55 --- /dev/null +++ b/scripts/remediate_heimdall.py @@ -0,0 +1,138 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +def main(): + print("=== Connecting to Shipyard Swarm Manager (140.44.4.71) for Heimdall Remediation ===") + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10) + + def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + + # 1. Ensure backup directory exists & take safety backup + run("mkdir -p /mnt/nfs_shares/backups") + run("cp -v /mnt/nfs_shares/heimdall/www/app.sqlite /mnt/nfs_shares/backups/heimdall_pre_remediation.sqlite") + + # 2. Database repair script executed locally on node + db_clean_script = ''' +import sqlite3 +import shutil + +db_path = "/mnt/nfs_shares/heimdall/www/app.sqlite" +con = sqlite3.connect(db_path) +cur = con.cursor() + +print("Initial total items:", cur.execute("SELECT count(*) FROM items").fetchone()[0]) + +# Delete all soft-deleted items (deleted_at IS NOT NULL) +cur.execute("DELETE FROM items WHERE deleted_at IS NOT NULL") +print("Deleted soft-deleted items. Remaining:", cur.execute("SELECT count(*) FROM items").fetchone()[0]) + +# Remove legacy Pi-hole items (Pi-hole 1 and Pi-hole 2) since Pi-hole stack is decommissioned +cur.execute("DELETE FROM items WHERE title LIKE 'Pi-hole%'") +print("Purged legacy Pi-hole items. Remaining:", cur.execute("SELECT count(*) FROM items").fetchone()[0]) + +# Deduplicate items by title just in case, keeping the highest ID +cur.execute(""" +DELETE FROM items +WHERE id NOT IN ( + SELECT MAX(id) + FROM items + GROUP BY title +) AND id != 0 +""") + +# Re-sequence the 'order' field sequentially +rows = cur.execute("SELECT id FROM items WHERE id != 0 ORDER BY [order] ASC, title ASC").fetchall() +for idx, (item_id,) in enumerate(rows): + cur.execute("UPDATE items SET [order] = ? WHERE id = ?", (idx, item_id)) + +con.commit() +cur.execute("REINDEX") +cur.execute("VACUUM") +con.close() + +print("Database sanitized, re-indexed, and vacuumed successfully!") +''' + sftp = c.open_sftp() + with sftp.file('/tmp/clean_heimdall_db.py', 'w') as f: + f.write(db_clean_script) + sftp.close() + + run("python3 /tmp/clean_heimdall_db.py") + + # 3. Create local high-performance storage directory on snshpd4lnxswm01 + run("mkdir -p /var/lib/docker-data/heimdall/config") + run("rsync -av --progress /mnt/nfs_shares/heimdall/ /var/lib/docker-data/heimdall/config/") + run("chown -R 1000:1000 /var/lib/docker-data/heimdall/config") + run("ls -la /var/lib/docker-data/heimdall/config/www/") + + # 4. Generate updated Docker Compose file with node pinning & local volume + new_compose = '''version: "3.8" + +services: + heimdall: + image: lscr.io/linuxserver/heimdall:latest + ports: + - target: 80 + published: 85 + mode: ingress + - target: 443 + published: 403 + mode: ingress + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/New_York + volumes: + - type: bind + source: /var/lib/docker-data/heimdall/config + target: /config + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - node.hostname == snshpd4lnxswm01 + restart_policy: + condition: on-failure + +networks: + default: + name: homelab_public + external: true +''' + sftp = c.open_sftp() + with sftp.file('/tmp/heimdall_compose.yml', 'w') as f: + f.write(new_compose) + sftp.close() + + # 5. Redeploy Heimdall stack + run("docker stack deploy -c /tmp/heimdall_compose.yml heimdall") + time.sleep(5) + run("docker stack ps heimdall") + + # 6. Setup daily atomic SQLite backup cron job + cron_cmd = '(crontab -l 2>/dev/null | grep -v "heimdall_backup"; echo "30 3 * * * python3 -c \\"import sqlite3; con=sqlite3.connect(\'/var/lib/docker-data/heimdall/config/www/app.sqlite\'); b=sqlite3.connect(\'/mnt/nfs_shares/backups/heimdall_daily.sqlite\'); con.backup(b); b.close(); con.close()\\" >/dev/null 2>&1") | crontab -' + run(cron_cmd) + run("crontab -l | grep heimdall") + + c.close() + print("\n=== Heimdall Storage & DB Remediation Complete ===") + +if __name__ == '__main__': + main() + diff --git a/scripts/remediate_swarm_services.py b/scripts/remediate_swarm_services.py new file mode 100644 index 0000000..47961a3 --- /dev/null +++ b/scripts/remediate_swarm_services.py @@ -0,0 +1,46 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +def main(): + print("=== Connecting to Shipyard Swarm Manager (140.44.4.71) ===") + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10) + except Exception as e: + print(f"Failed to connect to root@140.44.4.71: {e}") + print("Retrying with username=snarf...") + c.connect("140.44.4.71", username="snarf", key_filename=key_path, timeout=10) + + def run_cmd(cmd): + print(f"\n>>> Running: {cmd}") + stdin, stdout, stderr = c.exec_command(cmd) + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + if out: + print(out) + if err: + print(f"[stderr] {err}") + return out, err + + # 1. Inspect live Swarm stacks & services + run_cmd("docker stack ls") + run_cmd("docker service ls") + run_cmd("docker ps --filter 'name=heimdall' --filter 'name=pihole' --filter 'name=watchtower'") + + # 2. Check Heimdall mount and files + run_cmd("docker inspect $(docker ps -q -f name=heimdall) --format '{{json .Mounts}}' 2>/dev/null || true") + + # 3. Check Pi-hole stack status + run_cmd("docker stack ps pihole 2>/dev/null || true") + + c.close() + print("\n=== Inspection Complete ===") + +if __name__ == '__main__': + main() + diff --git a/scripts/remote_inspect_heimdall.py b/scripts/remote_inspect_heimdall.py new file mode 100644 index 0000000..1357b70 --- /dev/null +++ b/scripts/remote_inspect_heimdall.py @@ -0,0 +1,31 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +script = ''' +import sqlite3 + +con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite') +cur = con.cursor() +print("=== USERS ===") +for u in cur.execute("SELECT * FROM users").fetchall(): + print(u) + +print("\\n=== ITEMS (Total %d) ===" % cur.execute("SELECT count(*) FROM items").fetchone()[0]) +for r in cur.execute("SELECT id, user_id, title, url, pinned, [order], deleted_at FROM items ORDER BY id").fetchall(): + print(r) +''' + +sftp = c.open_sftp() +with sftp.file('/tmp/inspect_heimdall.py', 'w') as f: + f.write(script) +sftp.close() + +_, stdout, _ = c.exec_command('python3 /tmp/inspect_heimdall.py') +print(stdout.read().decode()) +c.close() + diff --git a/scripts/remove_pihole_stack.py b/scripts/remove_pihole_stack.py new file mode 100644 index 0000000..7096ec6 --- /dev/null +++ b/scripts/remove_pihole_stack.py @@ -0,0 +1,28 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +print("=== Removing unused Pihole Stack from Swarm ===") +run("docker stack rm pihole") +time.sleep(5) +run("docker service ls | grep pihole || echo 'No pihole services running'") +run("docker stack ls") + +c.close() + diff --git a/scripts/reset_grafana_pass.py b/scripts/reset_grafana_pass.py new file mode 100644 index 0000000..fbbc00e --- /dev/null +++ b/scripts/reset_grafana_pass.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, o, _ = c.exec_command('kubectl get pod -n default -l app.kubernetes.io/name=grafana -o jsonpath="{.items[0].metadata.name}"') +pod = o.read().decode().strip() +print("Grafana pod:", pod) + +_, o2, _ = c.exec_command(f'kubectl exec -n default {pod} -- grafana-cli admin reset-admin-password "Katzir476!!!"') +print(o2.read().decode()) + +c.close() + diff --git a/scripts/restart_homepage.py b/scripts/restart_homepage.py new file mode 100644 index 0000000..f840394 --- /dev/null +++ b/scripts/restart_homepage.py @@ -0,0 +1,25 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + if res: + print(res) + if err: + print("[stderr]", err) + return res + +run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure") +run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s") + +c.close() + diff --git a/scripts/restructure_homepage_cm.py b/scripts/restructure_homepage_cm.py new file mode 100644 index 0000000..72b1399 --- /dev/null +++ b/scripts/restructure_homepage_cm.py @@ -0,0 +1,34 @@ +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + text = f.read() + +docs = text.split('---') +# docs[0] = Deployment +# docs[1] = Service +# docs[2] = ConfigMap + +cm = docs[2] + +# Extract the footer +footer_marker = "kind: ConfigMap" +idx = cm.find(footer_marker) +if idx != -1: + footer = cm[idx:] + body = cm[:idx] + + # Remove apiVersion: v1 from body + body_clean = body.replace('apiVersion: v1\n', '') + + # Construct clean ConfigMap + new_cm = f""" +apiVersion: v1 +{footer.strip()} +{body_clean.strip()} +""" + docs[2] = new_cm + +new_text = '---'.join(docs) +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f: + f.write(new_text) + +print("Restructured homepage.yaml successfully!") + diff --git a/scripts/search_next_api.py b/scripts/search_next_api.py new file mode 100644 index 0000000..5f35979 --- /dev/null +++ b/scripts/search_next_api.py @@ -0,0 +1,15 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Search for homeassistant in .next:') +print(run(f'kubectl exec -n infrastructure {pod} -- grep -rn "homeassistant" /app/.next/server/pages/api/')) + +c.close() diff --git a/scripts/setup_argo_account.py b/scripts/setup_argo_account.py new file mode 100644 index 0000000..0a6699f --- /dev/null +++ b/scripts/setup_argo_account.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print(run('kubectl patch cm argocd-cm -n infrastructure --type merge -p \'{"data":{"accounts.homepage":"apiKey"}}\'')) +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=argocd-server -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl exec -n infrastructure {pod} -- argocd-server --version || true")) + +c.close() + diff --git a/scripts/setup_homepage_rbac.py b/scripts/setup_homepage_rbac.py new file mode 100644 index 0000000..785a66c --- /dev/null +++ b/scripts/setup_homepage_rbac.py @@ -0,0 +1,61 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + return out + +rbac = """apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: homepage-reader +rules: + - apiGroups: [""] + resources: ["nodes", "namespaces", "pods", "services", "endpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes", "pods"] + verbs: ["get", "list"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list"] + - apiGroups: ["traefik.containo.us", "traefik.io"] + resources: ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "traefikservices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: homepage-reader-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: homepage-reader +subjects: + - kind: ServiceAccount + name: default + namespace: infrastructure +""" + +stdin, stdout, stderr = c.exec_command('cat > /tmp/homepage_rbac.yaml') +stdin.write(rbac) +stdin.close() +stdout.read() + +run("kubectl apply -f /tmp/homepage_rbac.yaml") +run("kubectl auth can-i list ingresses.networking.k8s.io --as=system:serviceaccount:infrastructure:default") + +c.close() diff --git a/scripts/setup_kuma_tags_status_pages.py b/scripts/setup_kuma_tags_status_pages.py new file mode 100644 index 0000000..a8fd37b --- /dev/null +++ b/scripts/setup_kuma_tags_status_pages.py @@ -0,0 +1,298 @@ +import paramiko +import sys + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if err and "Warning" not in err: + print("[stderr]", err) + return out + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +print(f"Target Pod: {pod}") + +sql_script = """ +-- 1. Create Tags +INSERT IGNORE INTO tag (id, name, color) VALUES + (1, 'core', '#3b82f6'), + (2, 'edge', '#06b6d4'), + (3, 'gitops', '#10b981'), + (4, 'storage', '#f59e0b'), + (5, 'ai', '#8b5cf6'), + (6, 'smarthome', '#14b8a6'), + (7, 'DNS', '#6366f1'), + (8, 'VIP', '#ec4899'), + (9, 'IP', '#64748b'), + (10, 'Tier-1-Critical', '#ef4444'), + (11, 'Tier-2-Operational', '#3b82f6'), + (12, 'Tier-3-Dev-Tools', '#84cc16'); + +-- 2. Clear old monitor_tag mappings and re-seed cleanly +DELETE FROM monitor_tag; + +-- Monitor mappings: (monitor_id, tag_id) +-- 52: Gotify Hub (DNS) -> edge(2), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (52, 2), (52, 7), (52, 12); +-- 53: Heimdall Portal (DNS) -> edge(2), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (53, 2), (53, 7), (53, 12); +-- 54: NPM Admin Console (DNS) -> edge(2), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (54, 2), (54, 7), (54, 10); +-- 55: Authentik SSO (DNS) -> core(1), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (55, 1), (55, 7), (55, 10); +-- 56: Rancher Management (DNS) -> core(1), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (56, 1), (56, 7), (56, 10); +-- 57: Harvester Console (DNS) -> core(1), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (57, 1), (57, 7), (57, 11); +-- 58: Gitea DevOps (DNS) -> gitops(3), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (58, 3), (58, 7), (58, 11); +-- 59: ArgoCD GitOps (DNS) -> gitops(3), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (59, 3), (59, 7), (59, 11); +-- 60: AWX Automation (DNS) -> gitops(3), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (60, 3), (60, 7), (60, 12); +-- 61: Semaphore Ansible (DNS) -> gitops(3), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (61, 3), (61, 7), (61, 11); +-- 62: Jenkins CI/CD (DNS) -> gitops(3), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (62, 3), (62, 7), (62, 12); +-- 63: Zabbix Web UI (DNS) -> core(1), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (63, 1), (63, 7), (63, 11); +-- 64: Grafana Dashboards (DNS) -> core(1), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (64, 1), (64, 7), (64, 11); +-- 65: Longhorn Aerodrome (DNS) -> storage(4), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (65, 4), (65, 7), (65, 10); +-- 66: Longhorn Spaceport (DNS) -> storage(4), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (66, 4), (66, 7), (66, 10); +-- 67: Home Assistant (DNS) -> smarthome(6), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (67, 6), (67, 7), (67, 11); +-- 68: PeaNUT UPS Portal (DNS) -> smarthome(6), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (68, 6), (68, 7), (68, 11); +-- 69: Open WebUI AI (DNS) -> ai(5), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (69, 5), (69, 7), (69, 12); +-- 70: SnarfNAS DSM (DNS) -> storage(4), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (70, 4), (70, 7), (70, 10); +-- 71: SN-NAS DSM (DNS) -> storage(4), DNS(7), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (71, 4), (71, 7), (71, 10); +-- 72: Homepage (DNS) -> edge(2), DNS(7), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (72, 2), (72, 7), (72, 11); +-- 73: MeshCentral (DNS) -> edge(2), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (73, 2), (73, 7), (73, 12); +-- 74: Homebox (DNS) -> edge(2), DNS(7), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (74, 2), (74, 7), (74, 12); +-- 76: Swarm VIP - Heimdall (VIP) -> edge(2), VIP(8), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (76, 2), (76, 8), (76, 12); +-- 77: Swarm VIP - Homebox (VIP) -> edge(2), VIP(8), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (77, 2), (77, 8), (77, 12); +-- 78: Swarm VIP - PeaNUT (VIP) -> smarthome(6), VIP(8), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (78, 6), (78, 8), (78, 11); +-- 80: Aerodrome VIP - Authentik (VIP) -> core(1), VIP(8), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (80, 1), (80, 8), (80, 10); +-- 82: Aerodrome VIP - Grafana (VIP) -> core(1), VIP(8), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (82, 1), (82, 8), (82, 11); +-- 90: Spaceport Node - Longhorn (VIP) -> storage(4), VIP(8), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (90, 4), (90, 8), (90, 10); +-- 91: Home Assistant Core (IP) -> smarthome(6), IP(9), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (91, 6), (91, 9), (91, 10); +-- 92: AI Hailo Engine (IP) -> ai(5), IP(9), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (92, 5), (92, 9), (92, 12); +-- 93: AI Open WebUI (IP) -> ai(5), IP(9), Tier-3(12) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (93, 5), (93, 9), (93, 12); +-- 94: Harvester HCI Console (VIP) -> core(1), VIP(8), Tier-2(11) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (94, 1), (94, 8), (94, 11); +-- 95: Synology NAS 1 DSM (IP) -> storage(4), IP(9), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (95, 4), (95, 9), (95, 10); +-- 96: SnarfNAS DSM (IP) -> storage(4), IP(9), Tier-1(10) +INSERT INTO monitor_tag (monitor_id, tag_id) VALUES (96, 4), (96, 9), (96, 10); + +-- 3. Upsert Status Pages +-- Status Page 1: default (Unified) +INSERT INTO status_page (id, slug, title, description, icon, theme, published, show_tags, auto_refresh_interval) +VALUES (1, 'default', 'SnarfNet Infrastructure', 'Overall health of SnarfNet endpoints and services', '/icon.svg', 'dark', 1, 1, 60) +ON DUPLICATE KEY UPDATE + title='SnarfNet Unified Infrastructure', + description='Comprehensive health of all SnarfNet clusters, edge ingress, and services', + theme='dark', + published=1, + show_tags=1, + auto_refresh_interval=60; + +-- Status Page 2: core (Tier-1 Critical & VIPs) +INSERT INTO status_page (id, slug, title, description, icon, theme, published, show_tags, auto_refresh_interval) +VALUES (2, 'core', 'SnarfNet Core Infrastructure & VIPs', 'L4 VIPs, direct hardware probes, and Tier-1 foundational cluster services', '/icon.svg', 'dark', 1, 1, 60) +ON DUPLICATE KEY UPDATE + title='SnarfNet Core Infrastructure & VIPs', + description='L4 VIPs, direct hardware probes, and Tier-1 foundational cluster services', + theme='dark', + published=1, + show_tags=1, + auto_refresh_interval=60; + +-- Status Page 3: gitops (GitOps, DevOps & CI/CD) +INSERT INTO status_page (id, slug, title, description, icon, theme, published, show_tags, auto_refresh_interval) +VALUES (3, 'gitops', 'SnarfNet GitOps & Automation Pipeline', 'Continuous delivery reconcilers, repositories, playbooks, and build automation', '/icon.svg', 'dark', 1, 1, 60) +ON DUPLICATE KEY UPDATE + title='SnarfNet GitOps & Automation Pipeline', + description='Continuous delivery reconcilers, repositories, playbooks, and build automation', + theme='dark', + published=1, + show_tags=1, + auto_refresh_interval=60; + +-- Status Page 4: edge (Edge Ingress & Portals) +INSERT INTO status_page (id, slug, title, description, icon, theme, published, show_tags, auto_refresh_interval) +VALUES (4, 'edge', 'SnarfNet Edge & Public Endpoints', 'Reverse proxy routing, user-facing portals, and perimeter gateways', '/icon.svg', 'dark', 1, 1, 60) +ON DUPLICATE KEY UPDATE + title='SnarfNet Edge & Public Endpoints', + description='Reverse proxy routing, user-facing portals, and perimeter gateways', + theme='dark', + published=1, + show_tags=1, + auto_refresh_interval=60; + +-- 4. Clean and recreate Groups & Monitor Groups +DELETE FROM monitor_group; +DELETE FROM `group`; + +-- === PAGE 1 (default) GROUPS === +INSERT INTO `group` (id, name, status_page_id, weight, public, active) VALUES + (10, 'Core Identity & Cluster Control', 1, 1, 1, 1), + (11, 'GitOps & CI/CD Pipeline', 1, 2, 1, 1), + (12, 'Observability & Monitoring', 1, 3, 1, 1), + (13, 'Storage SAN & Replicated Volumes', 1, 4, 1, 1), + (14, 'Edge Ingress & User Portals', 1, 5, 1, 1), + (15, 'Smart Home & Edge AI', 1, 6, 1, 1); + +-- Map monitors for Page 1: +-- Group 10 (Core Identity & Cluster Control) +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (80, 10, 1), -- Aerodrome VIP - Authentik + (55, 10, 2), -- Authentik SSO DNS + (56, 10, 3), -- Rancher Management DNS + (94, 10, 4), -- Harvester HCI VIP + (57, 10, 5); -- Harvester Console DNS + +-- Group 11 (GitOps & CI/CD Pipeline) +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (59, 11, 1), -- ArgoCD GitOps DNS + (58, 11, 2), -- Gitea DevOps DNS + (61, 11, 3), -- Semaphore Ansible DNS + (60, 11, 4), -- AWX Automation DNS + (62, 11, 5); -- Jenkins CI/CD DNS + +-- Group 12 (Observability & Monitoring) +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (82, 12, 1), -- Aerodrome VIP - Grafana + (64, 12, 2), -- Grafana Dashboards DNS + (63, 12, 3); -- Zabbix Web UI DNS + +-- Group 13 (Storage SAN & Replicated Volumes) +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (90, 13, 1), -- Spaceport Node - Longhorn VIP + (66, 13, 2), -- Longhorn Spaceport DNS + (65, 13, 3), -- Longhorn Aerodrome DNS + (96, 13, 4), -- SnarfNAS DSM IP + (70, 13, 5), -- SnarfNAS DSM DNS + (95, 13, 6), -- Synology NAS 1 DSM IP + (71, 13, 7); -- SN-NAS DSM DNS + +-- Group 14 (Edge Ingress & User Portals) +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (54, 14, 1), -- NPM Admin Console DNS + (72, 14, 2), -- Homepage DNS + (76, 14, 3), -- Swarm VIP - Heimdall + (53, 14, 4), -- Heimdall Portal DNS + (52, 14, 5), -- Gotify Hub DNS + (77, 14, 6), -- Swarm VIP - Homebox + (74, 14, 7), -- Homebox DNS + (73, 14, 8); -- MeshCentral DNS + +-- Group 15 (Smart Home & Edge AI) +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (91, 15, 1), -- Home Assistant Core IP + (67, 15, 2), -- Home Assistant DNS + (78, 15, 3), -- Swarm VIP - PeaNUT + (68, 15, 4), -- PeaNUT UPS Portal DNS + (93, 15, 5), -- AI Open WebUI IP + (69, 15, 6), -- Open WebUI AI DNS + (92, 15, 7); -- AI Hailo Engine IP + + +-- === PAGE 2 (core) GROUPS === +INSERT INTO `group` (id, name, status_page_id, weight, public, active) VALUES + (20, 'Cluster VIPs & LoadBalancers', 2, 1, 1, 1), + (21, 'Hardware Node & SAN Storage Probes', 2, 2, 1, 1), + (22, 'Tier-1 Foundational Services', 2, 3, 1, 1); + +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (80, 20, 1), -- Aerodrome VIP - Authentik + (82, 20, 2), -- Aerodrome VIP - Grafana + (90, 20, 3), -- Spaceport Node - Longhorn VIP + (94, 20, 4), -- Harvester HCI Console VIP + (78, 20, 5), -- Swarm VIP - PeaNUT + (96, 21, 1), -- SnarfNAS DSM IP + (95, 21, 2), -- Synology NAS 1 DSM IP + (91, 21, 3), -- Home Assistant Core IP + (55, 22, 1), -- Authentik SSO DNS + (56, 22, 2), -- Rancher Management DNS + (65, 22, 3), -- Longhorn Aerodrome DNS + (66, 22, 4); -- Longhorn Spaceport DNS + + +-- === PAGE 3 (gitops) GROUPS === +INSERT INTO `group` (id, name, status_page_id, weight, public, active) VALUES + (30, 'Declarative GitOps & Repositories', 3, 1, 1, 1), + (31, 'Automation Controllers & Runners', 3, 2, 1, 1), + (32, 'CI/CD Pipelines & Dev Tools', 3, 3, 1, 1); + +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (59, 30, 1), -- ArgoCD GitOps DNS + (58, 30, 2), -- Gitea DevOps DNS + (61, 31, 1), -- Semaphore Ansible DNS + (60, 31, 2), -- AWX Automation DNS + (62, 32, 1); -- Jenkins CI/CD DNS + + +-- === PAGE 4 (edge) GROUPS === +INSERT INTO `group` (id, name, status_page_id, weight, public, active) VALUES + (40, 'Reverse Proxy & Ingress Gateways', 4, 1, 1, 1), + (41, 'Operational Dashboards & Portals', 4, 2, 1, 1), + (42, 'Remote Access & Notification Hubs', 4, 3, 1, 1); + +INSERT INTO monitor_group (monitor_id, group_id, weight) VALUES + (54, 40, 1), -- NPM Admin Console DNS + (55, 40, 2), -- Authentik SSO DNS + (72, 41, 1), -- Homepage DNS + (76, 41, 2), -- Swarm VIP - Heimdall + (53, 41, 3), -- Heimdall Portal DNS + (77, 41, 4), -- Swarm VIP - Homebox + (74, 41, 5), -- Homebox DNS + (52, 42, 1), -- Gotify Hub DNS + (73, 42, 2); -- MeshCentral DNS +""" + +stdin, stdout, stderr = c.exec_command(f"kubectl exec -n monitoring {pod} -i -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma") +stdin.write(sql_script) +stdin.close() + +out = stdout.read().decode() +err = stderr.read().decode() + +if out: + print("[Output]", out) +if err: + print("[Errors]", err) + +print("\n=== VERIFYING STATUS PAGES ===") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, slug, title, show_tags, published FROM status_page;" """)) + +print("\n=== VERIFYING TAG COUNTS ===") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT t.name, t.color, count(mt.monitor_id) AS total_monitors FROM tag t LEFT JOIN monitor_tag mt ON t.id = mt.tag_id GROUP BY t.id;" """)) + +print("\n=== VERIFYING GROUP COUNTS PER STATUS PAGE ===") +print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT sp.slug, g.name AS group_name, count(mg.monitor_id) AS monitors FROM status_page sp JOIN \`group\` g ON sp.id = g.status_page_id LEFT JOIN monitor_group mg ON g.id = mg.group_id GROUP BY g.id ORDER BY sp.id, g.weight;" """)) + +c.close() + diff --git a/scripts/sync_argocd_app.py b/scripts/sync_argocd_app.py new file mode 100644 index 0000000..221d79d --- /dev/null +++ b/scripts/sync_argocd_app.py @@ -0,0 +1,44 @@ +import requests +import json +import time +import urllib3 +urllib3.disable_warnings() + +token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0" +headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" +} + +# 1. Hard refresh aerodrome-grafana +url_refresh = "https://argo.snarfnet.net/api/v1/applications/aerodrome-grafana?refresh=hard" +resp = requests.get(url_refresh, headers=headers, verify=False, timeout=15) +print("Hard refresh status:", resp.status_code) + +time.sleep(3) + +# 2. Trigger sync +url_sync = "https://argo.snarfnet.net/api/v1/applications/aerodrome-grafana/sync" +sync_payload = { + "revision": "HEAD", + "prune": False, + "strategy": {"hook": {"force": True}} +} +resp_sync = requests.post(url_sync, headers=headers, json=sync_payload, verify=False, timeout=15) +print("Sync trigger status:", resp_sync.status_code) +if resp_sync.status_code not in [200, 201]: + print("Response:", resp_sync.text) + +time.sleep(5) + +# 3. Check status +url_app = "https://argo.snarfnet.net/api/v1/applications/aerodrome-grafana" +resp_app = requests.get(url_app, headers=headers, verify=False, timeout=10) +app_data = resp_app.json() + +print("\n=== APPLICATION STATUS ===") +print("Sync Status:", app_data.get("status", {}).get("sync", {}).get("status")) +print("Health Status:", app_data.get("status", {}).get("health", {}).get("status")) +print("Operation Phase:", app_data.get("status", {}).get("operationState", {}).get("phase")) +print("Operation Message:", app_data.get("status", {}).get("operationState", {}).get("message")) + diff --git a/scripts/sync_homepage.py b/scripts/sync_homepage.py new file mode 100644 index 0000000..df99202 --- /dev/null +++ b/scripts/sync_homepage.py @@ -0,0 +1,23 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if out: + print(out) + if err: + print("[stderr]", err) + return out + +run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure") +run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s") + +c.close() + diff --git a/scripts/test_aerodrome_prom.py b/scripts/test_aerodrome_prom.py new file mode 100644 index 0000000..493e165 --- /dev/null +++ b/scripts/test_aerodrome_prom.py @@ -0,0 +1,6 @@ +import requests + +r = requests.get('http://140.44.4.11:30090/api/v1/query?query=up') +print("Status:", r.status_code) +print("Targets up:", len(r.json().get('data', {}).get('result', []))) + diff --git a/scripts/test_ai_metrics.py b/scripts/test_ai_metrics.py new file mode 100644 index 0000000..e17ccc9 --- /dev/null +++ b/scripts/test_ai_metrics.py @@ -0,0 +1,20 @@ +import requests + +endpoints = [ + ("vLLM Qwen (30800)", "http://140.44.4.84:30800/metrics"), + ("vLLM Gemma4 (30804)", "http://140.44.4.82:30804/metrics"), + ("vLLM DeepSeek-R1 (30805)", "http://140.44.4.81:30805/metrics"), + ("Triton (30803)", "http://140.44.4.83:30803/metrics"), + ("Qdrant (30633)", "http://140.44.4.82:30633/metrics"), +] + +for name, url in endpoints: + try: + r = requests.get(url, timeout=5) + print(f"{name}: HTTP {r.status_code}, length={len(r.text)}") + sample_metrics = [line for line in r.text.split('\n') if line and not line.startswith('#')][:5] + for sm in sample_metrics: + print(f" {sm}") + except Exception as e: + print(f"{name}: Failed -> {e}") + diff --git a/scripts/test_argo_api.py b/scripts/test_argo_api.py new file mode 100644 index 0000000..08a93d7 --- /dev/null +++ b/scripts/test_argo_api.py @@ -0,0 +1,8 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0" +headers = {"Authorization": f"Bearer {token}"} +r = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False) +print("Argo apps with homepage token:", r.status_code, len(r.json().get('items', []))) diff --git a/scripts/test_audit_devices.py b/scripts/test_audit_devices.py new file mode 100644 index 0000000..a5c844d --- /dev/null +++ b/scripts/test_audit_devices.py @@ -0,0 +1,19 @@ +import subprocess +import json + +devices = [ + ("Dell PowerEdge Node 4 iDRAC", "140.44.44.124"), + ("UniFi Aggregation Switch", "14.44.4.26"), + ("UniFi Closet Switch", "14.44.4.61"), + ("UniFi Compute Switch", "14.44.4.107"), + ("UniFi Attic Switch", "14.44.4.98"), + ("UniFi Garage Switch", "14.44.4.96"), + ("Tempest Weather Station", "14.4.4.47"), + ("Sense Energy Gateway", "14.4.4.48"), +] + +for name, ip in devices: + res = subprocess.run(["ping", "-n", "1", "-w", "1000", ip], capture_output=True, text=True) + status = "ONLINE" if res.returncode == 0 else "OFFLINE" + print(f"[{status}] {name} ({ip})") + diff --git a/scripts/test_cluster_widgets.py b/scripts/test_cluster_widgets.py new file mode 100644 index 0000000..cdb1797 --- /dev/null +++ b/scripts/test_cluster_widgets.py @@ -0,0 +1,76 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +headers = {"User-Agent": "Mozilla/5.0"} + +print("=== 1. Spaceport Kubernetes Cluster Metrics ===") +try: + r = requests.get("https://homepage.snarfnet.net/api/widgets/kubernetes", headers=headers, verify=False, timeout=10) + data = r.json() + cpu = data.get("cluster", {}).get("cpu", {}) + mem = data.get("cluster", {}).get("memory", {}) + print(f"Status: {r.status_code}") + print(f" CPU Load: {cpu.get('load', 0):.2f} / {cpu.get('total')} cores ({cpu.get('percent', 0):.1f}%)") + print(f" Memory: {mem.get('used', 0)/(1024**3):.1f} / {mem.get('total', 0)/(1024**3):.1f} GiB ({mem.get('percent', 0):.1f}%)") + print(f" Nodes: {len(data.get('nodes', []))} nodes online") +except Exception as e: + print("Error:", e) + +print("\n=== 2. Longhorn Spaceport Storage Metrics ===") +try: + r = requests.get("https://homepage.snarfnet.net/api/widgets/longhorn", headers=headers, verify=False, timeout=10) + data = r.json() + for n in data.get("nodes", []): + if n.get("id") == "total": + max_gb = n.get("maximum", 0) / (1024**3) + avail_gb = n.get("available", 0) / (1024**3) + used_gb = max_gb - avail_gb + print(f"Status: {r.status_code}") + print(f" Storage Total: {max_gb:.1f} GiB ({max_gb/1024:.2f} TiB)") + print(f" Storage Used: {used_gb:.1f} GiB ({used_gb/max_gb*100:.1f}%)") + print(f" Storage Available: {avail_gb:.1f} GiB") +except Exception as e: + print("Error:", e) + +print("\n=== 3. Portainer Swarm Metrics ===") +try: + url = "https://homepage.snarfnet.net/api/services/proxy?group=Storage%20%26%20Clusters&service=Portainer%20Swarm&index=0&endpoint=docker/containers" + r = requests.get(url, headers=headers, verify=False, timeout=10) + print(f"Status: {r.status_code}") + if r.status_code == 200: + containers = r.json() + running = len([c for c in containers if c.get("State") == "running"]) + stopped = len([c for c in containers if c.get("State") == "exited"]) + print(f" Containers: Running: {running}, Stopped: {stopped}, Total: {len(containers)}") +except Exception as e: + print("Error:", e) + +print("\n=== 4. Rancher (Aerodrome Cluster) Metrics ===") +try: + url = "https://homepage.snarfnet.net/api/services/proxy?group=Storage%20%26%20Clusters&service=Rancher%20(Aerodrome)&index=0" + r = requests.get(url, headers=headers, verify=False, timeout=10) + print(f"Status: {r.status_code}") + if r.status_code == 200: + d = r.json() + cap = d.get("capacity", {}) + print(f" Cluster State: {d.get('state')}, CPU Cores: {cap.get('cpu')}, Max Pods: {cap.get('pods')}") + else: + print(r.text[:200]) +except Exception as e: + print("Error:", e) + +print("\n=== 5. Harvester HCI (Spacedock Cluster) Metrics ===") +try: + url = "https://homepage.snarfnet.net/api/services/proxy?group=Storage%20%26%20Clusters&service=Harvester%20HCI%20(Spacedock)&index=0" + r = requests.get(url, headers=headers, verify=False, timeout=10) + print(f"Status: {r.status_code}") + if r.status_code == 200: + d = r.json() + cap = d.get("capacity", {}) + print(f" HCI State: {d.get('state')}, CPU Cores: {cap.get('cpu')}, Max Pods: {cap.get('pods')}") + else: + print(r.text[:200]) +except Exception as e: + print("Error:", e) diff --git a/scripts/test_dcgm.py b/scripts/test_dcgm.py new file mode 100644 index 0000000..adb0b3c --- /dev/null +++ b/scripts/test_dcgm.py @@ -0,0 +1,9 @@ +import requests + +for ip in ["140.44.4.81", "140.44.4.82", "140.44.4.83", "140.44.4.84"]: + try: + r = requests.get(f"http://{ip}:9400/metrics", timeout=2) + print(f"{ip}:9400 -> HTTP {r.status_code}") + except Exception as e: + print(f"{ip}:9400 -> Not reachable: {e}") + diff --git a/scripts/test_dcgm_metrics.py b/scripts/test_dcgm_metrics.py new file mode 100644 index 0000000..cfbda32 --- /dev/null +++ b/scripts/test_dcgm_metrics.py @@ -0,0 +1,12 @@ +import requests + +for ip in ["140.44.4.81", "140.44.4.82", "140.44.4.83", "140.44.4.84"]: + try: + r = requests.get(f"http://{ip}:9400/metrics", timeout=3) + print(f"=== {ip}:9400 -> HTTP {r.status_code}, len={len(r.text)} ===") + gpu_metrics = [line for line in r.text.split('\n') if 'DCGM_FI_DEV_GPU_UTIL' in line or 'DCGM_FI_DEV_FB_USED' in line or 'DCGM_FI_DEV_GPU_TEMP' in line] + for m in gpu_metrics[:3]: + print(" ", m) + except Exception as e: + print(f"{ip}:9400 -> error: {e}") + diff --git a/scripts/test_dcgm_nvml.py b/scripts/test_dcgm_nvml.py new file mode 100644 index 0000000..4cc59f0 --- /dev/null +++ b/scripts/test_dcgm_nvml.py @@ -0,0 +1,74 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'") + out = stdout.read().decode().strip() + return out + +manifest = """ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dcgm-exporter + namespace: ai-stack + labels: + app.kubernetes.io/name: dcgm-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + template: + metadata: + labels: + app.kubernetes.io/name: dcgm-exporter + spec: + hostNetwork: true + containers: + - name: dcgm-exporter + image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04 + env: + - name: DCGM_EXPORTER_LISTEN + value: ":9400" + - name: DCGM_EXPORTER_KUBERNETES + value: "false" + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64" + securityContext: + privileged: true + ports: + - name: metrics + containerPort: 9400 + volumeMounts: + - name: dev + mountPath: /dev + - name: nvidia-lib + mountPath: /usr/local/nvidia/lib64 + readOnly: true + volumes: + - name: dev + hostPath: + path: /dev + - name: nvidia-lib + hostPath: + path: /usr/local/nvidia/lib64 +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/dcgm.yaml") +stdin.write(manifest) +stdin.close() +stdout.channel.recv_exit_status() + +print("Applying DCGM Exporter with NVML mounts:") +print(run("kubectl apply -f /tmp/dcgm.yaml")) +time.sleep(10) +pod = run("kubectl get pod -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o jsonpath='{.items[0].metadata.name}'") +print("Logs from", pod) +print(run(f"kubectl logs -n ai-stack {pod} --tail=25")) + +c.close() + diff --git a/scripts/test_dcgm_pull.py b/scripts/test_dcgm_pull.py new file mode 100644 index 0000000..9c02f03 --- /dev/null +++ b/scripts/test_dcgm_pull.py @@ -0,0 +1,12 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'") + return stdout.read().decode().strip() + +print(run("crictl images | grep dcgm")) +c.close() diff --git a/scripts/test_deploy_dcgm.py b/scripts/test_deploy_dcgm.py new file mode 100644 index 0000000..a3943b8 --- /dev/null +++ b/scripts/test_deploy_dcgm.py @@ -0,0 +1,59 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'") + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +manifest = """ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dcgm-exporter + namespace: ai-stack + labels: + app.kubernetes.io/name: dcgm-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + template: + metadata: + labels: + app.kubernetes.io/name: dcgm-exporter + spec: + hostNetwork: true + containers: + - name: dcgm-exporter + image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04 + env: + - name: DCGM_EXPORTER_LISTEN + value: ":9400" + - name: DCGM_EXPORTER_KUBERNETES + value: "false" + securityContext: + privileged: true + ports: + - name: metrics + containerPort: 9400 +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/dcgm.yaml") +stdin.write(manifest) +stdin.close() +stdout.channel.recv_exit_status() + +print("Applying DCGM Exporter:") +print(run("kubectl apply -f /tmp/dcgm.yaml")) +import time +time.sleep(5) +print("Pods:") +print(run("kubectl get pods -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o wide")) + +c.close() + diff --git a/scripts/test_from_host.py b/scripts/test_from_host.py new file mode 100644 index 0000000..7d3f6ad --- /dev/null +++ b/scripts/test_from_host.py @@ -0,0 +1,25 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +print("DNS for harvester.spacedock.snarfnet.net:") +print(run("dig +short harvester.spacedock.snarfnet.net || getent hosts harvester.spacedock.snarfnet.net")) + +print("DNS for harvester.snarfnet.net:") +print(run("dig +short harvester.snarfnet.net || getent hosts harvester.snarfnet.net")) + +print("Jenkins HTTP status:") +print(run("curl -k -I https://jenkins.snarfnet.net")) + +print("NUT 14.4.4.44:3493 test:") +print(run("nc -z -v -w 2 14.4.4.44 3493 || true")) + +c.close() + diff --git a/scripts/test_gitea_widget.py b/scripts/test_gitea_widget.py new file mode 100644 index 0000000..7895a8c --- /dev/null +++ b/scripts/test_gitea_widget.py @@ -0,0 +1,9 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +headers = {"Authorization": "token f19e772cf54e40ceb2befe452e2bc24dd81fdb71"} +url = "https://git.snarfnet.net/api/v1/user" + +r = requests.get(url, headers=headers, verify=False, timeout=5) +print("Gitea user:", r.status_code, r.json().get('username')) diff --git a/scripts/test_glances_endpoint.py b/scripts/test_glances_endpoint.py new file mode 100644 index 0000000..281342b --- /dev/null +++ b/scripts/test_glances_endpoint.py @@ -0,0 +1,15 @@ +import requests + +for v in [3, 4]: + try: + r = requests.get(f"http://140.44.4.11:61208/api/{v}/quicklook", timeout=3) + print(f"Glances v{v} status:", r.status_code, r.text[:200]) + except Exception as e: + print(f"Glances v{v} err:", e) + + try: + r = requests.get(f"http://140.44.4.11:61208/api/{v}/cpu", timeout=3) + print(f"Glances v{v} cpu:", r.status_code, r.text[:200]) + except Exception as e: + print(f"Glances v{v} cpu err:", e) + diff --git a/scripts/test_gotify_health.py b/scripts/test_gotify_health.py new file mode 100644 index 0000000..7abf0e4 --- /dev/null +++ b/scripts/test_gotify_health.py @@ -0,0 +1,16 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +try: + r = requests.get('http://140.44.4.70:8090/health', timeout=5) + print("Gotify VIP /health:", r.status_code, r.text) +except Exception as e: + print("Gotify VIP error:", e) + +try: + r = requests.get('https://gotify.snarfnet.net/health', verify=False, timeout=5) + print("Gotify DNS /health:", r.status_code, r.text) +except Exception as e: + print("Gotify DNS error:", e) + diff --git a/scripts/test_grafana_admin.py b/scripts/test_grafana_admin.py new file mode 100644 index 0000000..a4ba98c --- /dev/null +++ b/scripts/test_grafana_admin.py @@ -0,0 +1,8 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +auth = ('admin', 'Katzir476!!!') +r = requests.get("https://grafana.snarfnet.net/api/admin/stats", auth=auth, verify=False) +print("Admin stats with user admin:", r.status_code, r.text) + diff --git a/scripts/test_grafana_sa.py b/scripts/test_grafana_sa.py new file mode 100644 index 0000000..14842f3 --- /dev/null +++ b/scripts/test_grafana_sa.py @@ -0,0 +1,14 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +auth = ('admin', 'Katzir476!!!') +sa_url = "https://grafana.snarfnet.net/api/serviceaccounts" +r_sa = requests.post(sa_url, auth=auth, json={"name": "homepage-viewer", "role": "Viewer"}, verify=False) +print("Create SA:", r_sa.status_code, r_sa.text) +if r_sa.status_code in (200, 201): + sa_id = r_sa.json().get('id') + tok_url = f"https://grafana.snarfnet.net/api/serviceaccounts/{sa_id}/tokens" + r_tok = requests.post(tok_url, auth=auth, json={"name": "homepage-key"}, verify=False) + print("Token created:", r_tok.status_code, r_tok.json()) diff --git a/scripts/test_grafana_token.py b/scripts/test_grafana_token.py new file mode 100644 index 0000000..e76920f --- /dev/null +++ b/scripts/test_grafana_token.py @@ -0,0 +1,11 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +headers = {"Authorization": "Bearer glsa_71Y8I8DSZt3nXUNFzygnmBcEF4FUmWdw_1f03e67c"} + +for ep in ["stats", "admin/stats", "search", "datasources", "alerts"]: + r = requests.get(f"https://grafana.snarfnet.net/api/{ep}", headers=headers, verify=False) + print(f"{ep}: status={r.status_code}", r.text[:150]) + diff --git a/scripts/test_grafana_yaml.py b/scripts/test_grafana_yaml.py new file mode 100644 index 0000000..f190fbf --- /dev/null +++ b/scripts/test_grafana_yaml.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("File size /tmp/grafana_full.yaml:", run("ls -l /tmp/grafana_full.yaml")) +print("Testing kubectl apply with --dry-run=client:") +print(run("kubectl apply -f /tmp/grafana_full.yaml --dry-run=client")) + +c.close() + diff --git a/scripts/test_group_sql.py b/scripts/test_group_sql.py new file mode 100644 index 0000000..1d28cf8 --- /dev/null +++ b/scripts/test_group_sql.py @@ -0,0 +1,20 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if err: + print("ERR:", err) + return out + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") + +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'INSERT INTO monitor_group (monitor_id, group_id, weight) SELECT id, 1, 1000 FROM monitor WHERE active=1;'")) +print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT count(id) FROM monitor_group;'")) + +c.close() diff --git a/scripts/test_ha_template.py b/scripts/test_ha_template.py new file mode 100644 index 0000000..3df4d63 --- /dev/null +++ b/scripts/test_ha_template.py @@ -0,0 +1,22 @@ +import requests + +url = "http://14.4.4.20:8123/api/template" +key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4" + +headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json" +} + +tests = [ + "{{ states('sensor.st_00158555_temperature') | float | round(1) }} Ā°F", + "{{ '%.1f' | format(states('sensor.st_00158555_temperature') | float) }} Ā°F", + "{{ states('sensor.st_00158555_wind_speed') | float | round(1) }} mph", + "{{ '%.1f' | format(states('sensor.st_00158555_wind_speed') | float) }} mph", + "{{ states('sensor.st_00158555_humidity') | float | round(1) }} %", + "{{ '%.1f' | format(states('sensor.st_00158555_humidity') | float) }} %", +] + +for t in tests: + res = requests.post(url, headers=headers, json={"template": t}) + print(f"{t} => {res.text}") diff --git a/scripts/test_homepage_backends.py b/scripts/test_homepage_backends.py new file mode 100644 index 0000000..ce5d567 --- /dev/null +++ b/scripts/test_homepage_backends.py @@ -0,0 +1,31 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def test_url(name, cmd): + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + err = e.read().decode().strip() + print(f"\n--- {name} ---") + if res: + print(res[:300]) + if err: + print("[stderr]", err[:200]) + +test_url("Gitea direct VIP", "curl -s -I http://140.44.4.62:90/api/v1/user -H 'Authorization: token 47c10c54574131c007302b246352f653facb5062'") +test_url("Gitea DNS", "curl -k -s -I https://git.snarfnet.net/api/v1/user -H 'Authorization: token 47c10c54574131c007302b246352f653facb5062'") +test_url("PeaNUT direct VIP", "curl -s -I http://140.44.4.70:8050/api/v1/devices") +test_url("PeaNUT DNS", "curl -k -s -I https://peanut.snarfnet.net/api/v1/devices") +test_url("Uptime Kuma direct VIP", "curl -s -I http://140.44.4.28:448/api/status-page/public") +test_url("Uptime Kuma DNS", "curl -k -s -I https://uptime.snarfnet.net/api/status-page/public") +test_url("NPM DNS", "curl -k -s -I https://nginx.snarfnet.net/api/") +test_url("Glances 140.44.4.11:61208", "curl -s -I http://140.44.4.11:61208/api/4/cpu") +test_url("Longhorn Aerodrome direct", "curl -s -I http://140.44.4.36:80/v1") +test_url("Longhorn Aerodrome DNS", "curl -k -s -I https://longhorn.aerodrome.snarfnet.net/v1") + +c.close() + diff --git a/scripts/test_homepage_http.py b/scripts/test_homepage_http.py new file mode 100644 index 0000000..bc95fe7 --- /dev/null +++ b/scripts/test_homepage_http.py @@ -0,0 +1,11 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +try: + resp = requests.get('https://homepage.snarfnet.net', verify=False, timeout=10) + print("Status:", resp.status_code) + print("Length:", len(resp.text)) +except Exception as e: + print("Error:", e) + diff --git a/scripts/test_homepage_static.py b/scripts/test_homepage_static.py new file mode 100644 index 0000000..c98f737 --- /dev/null +++ b/scripts/test_homepage_static.py @@ -0,0 +1,13 @@ +import urllib.request, ssl + +ctx = ssl._create_unverified_context() +for path in ['/topology.png', '/topology.html']: + url = 'https://homepage.snarfnet.net' + path + try: + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, context=ctx, timeout=10) as r: + data = r.read() + print(f'{url} -> HTTP {r.status}, length: {len(data)} bytes, Content-Type: {r.headers.get("Content-Type")}') + except Exception as e: + print(f'{url} -> Error: {e}') + diff --git a/scripts/test_homepage_tempest_api.py b/scripts/test_homepage_tempest_api.py new file mode 100644 index 0000000..bc4bbc2 --- /dev/null +++ b/scripts/test_homepage_tempest_api.py @@ -0,0 +1,19 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +# Let's test the widget API endpoint on homepage +url = "https://homepage.snarfnet.net/api/services/proxy?group=Environment%20%26%20Weather&service=Tempest%20Weather%20Station&index=0" + +headers = { + "User-Agent": "Mozilla/5.0" +} + +try: + r = requests.get(url, headers=headers, verify=False, timeout=10) + print("Status code:", r.status_code) + print("Response body:") + print(json.dumps(r.json(), indent=2)) +except Exception as e: + print("Error:", e) diff --git a/scripts/test_info_widgets.py b/scripts/test_info_widgets.py new file mode 100644 index 0000000..c3256f4 --- /dev/null +++ b/scripts/test_info_widgets.py @@ -0,0 +1,16 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +headers = {"User-Agent": "Mozilla/5.0"} + +for ep in ["kubernetes", "longhorn"]: + url = f"https://homepage.snarfnet.net/api/widgets/{ep}" + try: + r = requests.get(url, headers=headers, verify=False, timeout=10) + print(f"=== {ep} (status {r.status_code}) ===") + print(json.dumps(r.json(), indent=2)[:500]) + except Exception as e: + print(f"=== {ep} ERROR ===", e) + diff --git a/scripts/test_kubelet_token.py b/scripts/test_kubelet_token.py new file mode 100644 index 0000000..13a26b6 --- /dev/null +++ b/scripts/test_kubelet_token.py @@ -0,0 +1,19 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +test_script = """ +TOKEN=$(kubectl create token default --duration=1h) +echo "Token length: ${#TOKEN}" +curl -sk -H "Authorization: Bearer $TOKEN" https://localhost:6443/api/v1/nodes/sninfr2lnxctl01/proxy/metrics/cadvisor | head -n 15 +""" + +print(run(test_script)) +c.close() + diff --git a/scripts/test_kuma_failing_monitors.py b/scripts/test_kuma_failing_monitors.py new file mode 100644 index 0000000..29d02f6 --- /dev/null +++ b/scripts/test_kuma_failing_monitors.py @@ -0,0 +1,23 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + return out + +pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64" +print("Harvester DNS test:") +print(run(f"kubectl exec -n monitoring {pod} -- nslookup harvester.spacedock.snarfnet.net || nslookup harvester.snarfnet.net || true")) + +print("\nJenkins HTTP test:") +print(run(f"kubectl exec -n monitoring {pod} -- wget -S -O /dev/null https://jenkins.snarfnet.net 2>&1 | head -n 15")) + +print("\nNUT test:") +print(run(f"kubectl exec -n monitoring {pod} -- nc -zv 14.4.4.44 3493 || nc -zv 140.44.4.44 3493 || true")) + +c.close() + diff --git a/scripts/test_kuma_internal.py b/scripts/test_kuma_internal.py new file mode 100644 index 0000000..f8652c5 --- /dev/null +++ b/scripts/test_kuma_internal.py @@ -0,0 +1,9 @@ +import requests + +for url in ["http://140.44.4.28:448", "http://140.44.4.11:30010", "http://140.44.4.21:30010"]: + try: + r = requests.get(f"{url}/api/status-page/default", timeout=3) + print(url, "status:", r.status_code, r.text[:100]) + except Exception as e: + print(url, "err:", e) + diff --git a/scripts/test_kuma_live.py b/scripts/test_kuma_live.py new file mode 100644 index 0000000..a217c80 --- /dev/null +++ b/scripts/test_kuma_live.py @@ -0,0 +1,8 @@ +import requests + +r1 = requests.get("http://140.44.4.28:448/api/status-page/default") +print("status-page:", r1.status_code, r1.json().get('config', {}).get('title')) + +r2 = requests.get("http://140.44.4.28:448/api/status-page/heartbeat/default") +print("heartbeat:", r2.status_code, len(r2.json().get('heartbeatList', {}))) + diff --git a/scripts/test_kuma_slug.py b/scripts/test_kuma_slug.py new file mode 100644 index 0000000..f3a5e49 --- /dev/null +++ b/scripts/test_kuma_slug.py @@ -0,0 +1,9 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +r = requests.get("https://uptime.snarfnet.net/api/status-page/heartbeat/default", verify=False) +print("text:", r.text[:300]) + +r2 = requests.get("https://uptime.snarfnet.net/api/status-page/default", verify=False) +print("status-page:", r2.text[:300]) diff --git a/scripts/test_kuma_status_pages.py b/scripts/test_kuma_status_pages.py new file mode 100644 index 0000000..2d67bb9 --- /dev/null +++ b/scripts/test_kuma_status_pages.py @@ -0,0 +1,20 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +slugs = ['default', 'core', 'gitops', 'edge'] +base = "http://140.44.4.28:448/api/status-page" + +for slug in slugs: + try: + url = f"{base}/{slug}" + r = requests.get(url, verify=False, timeout=5) + data = r.json() + config = data.get("config", {}) + groups = data.get("publicGroupList", []) + total_m = sum(len(g.get("monitorList", [])) for g in groups) + print(f"[{slug}] Status: {r.status_code} | Title: '{config.get('title')}' | Groups: {len(groups)} | Monitors: {total_m} | ShowTags: {config.get('showTags')}") + except Exception as e: + print(f"[{slug}] Error: {e}") + diff --git a/scripts/test_peanut.py b/scripts/test_peanut.py new file mode 100644 index 0000000..e1906f4 --- /dev/null +++ b/scripts/test_peanut.py @@ -0,0 +1,7 @@ +import requests + +try: + r = requests.get("http://140.44.4.71:8050/api/v1/devices", timeout=3) + print("PeaNUT on 71:", r.status_code, r.json()) +except Exception as e: + print("PeaNUT on 71 err:", e) diff --git a/scripts/test_pod_ha.py b/scripts/test_pod_ha.py new file mode 100644 index 0000000..6f9161e --- /dev/null +++ b/scripts/test_pod_ha.py @@ -0,0 +1,18 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + return out + ("\nERR: " + err if err else "") + +pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"') +print('Pod:', pod) +print('Testing wget -S:') +print(run(f'kubectl exec -n infrastructure {pod} -- wget -S -O /dev/null http://14.4.4.20:8123/api/')) + +c.close() diff --git a/scripts/test_portainer_auth.py b/scripts/test_portainer_auth.py new file mode 100644 index 0000000..8ce2b65 --- /dev/null +++ b/scripts/test_portainer_auth.py @@ -0,0 +1,13 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +url = "https://140.44.4.71:9443/api/auth" +res = requests.post(url, json={"username": "admin", "password": "Cfxg0GqDfocWcteE"}, verify=False) +jwt = res.json().get("jwt") +headers = {"Authorization": f"Bearer {jwt}"} + +r_create = requests.post("https://140.44.4.71:9443/api/users/1/tokens", headers=headers, json={"description": "homepage-dashboard", "password": "Cfxg0GqDfocWcteE"}, verify=False) +print("Create token status:", r_create.status_code) +print("Token response:", r_create.json()) diff --git a/scripts/test_portainer_endpoint.py b/scripts/test_portainer_endpoint.py new file mode 100644 index 0000000..c0a12b4 --- /dev/null +++ b/scripts/test_portainer_endpoint.py @@ -0,0 +1,17 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +headers = {"User-Agent": "Mozilla/5.0"} +url = "https://homepage.snarfnet.net/api/services/proxy?group=Storage%20%26%20Clusters&service=Portainer%20Swarm&index=0&endpoint=docker/containers" + +r = requests.get(url, headers=headers, verify=False, timeout=10) +print(f"Status: {r.status_code}") +if r.status_code == 200: + containers = r.json() + running = len([c for c in containers if c.get("State") == "running"]) + stopped = len([c for c in containers if c.get("State") == "exited"]) + print(f"Containers - Running: {running}, Stopped: {stopped}, Total: {len(containers)}") +else: + print(r.text) + diff --git a/scripts/test_portainer_key.py b/scripts/test_portainer_key.py new file mode 100644 index 0000000..c58e3b0 --- /dev/null +++ b/scripts/test_portainer_key.py @@ -0,0 +1,15 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +headers = {"X-API-Key": "ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8="} +url = "https://140.44.4.71:9443/api/endpoints/3/docker/containers/json?all=1" + +r = requests.get(url, headers=headers, verify=False) +print("Portainer containers status:", r.status_code) +if r.status_code == 200: + data = r.json() + running = len([c for c in data if c.get("State") == "running"]) + stopped = len([c for c in data if c.get("State") == "exited"]) + print(f"Containers: Running: {running}, Stopped: {stopped}, Total: {len(data)}") + diff --git a/scripts/test_prom_data.py b/scripts/test_prom_data.py new file mode 100644 index 0000000..f3d14ee --- /dev/null +++ b/scripts/test_prom_data.py @@ -0,0 +1,14 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Node count:", run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(node_load1)'")) +print("Container count:", run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(container_cpu_usage_seconds_total)'")) +c.close() + diff --git a/scripts/test_prom_rbac.py b/scripts/test_prom_rbac.py new file mode 100644 index 0000000..5711d36 --- /dev/null +++ b/scripts/test_prom_rbac.py @@ -0,0 +1,56 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +test_script = """ +cat << 'EOF' | kubectl apply -f - +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: default +EOF + +TOKEN=$(kubectl create token prometheus --duration=1h) +curl -sk -H "Authorization: Bearer $TOKEN" https://localhost:6443/api/v1/nodes/sninfr2lnxctl01/proxy/metrics/cadvisor | head -n 15 +""" + +print(run(test_script)) +c.close() + diff --git a/scripts/test_pull_node_exporter.py b/scripts/test_pull_node_exporter.py new file mode 100644 index 0000000..6330a80 --- /dev/null +++ b/scripts/test_pull_node_exporter.py @@ -0,0 +1,14 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +print("Testing crictl pull prom/node-exporter:v1.8.2:") +print(run("crictl pull prom/node-exporter:v1.8.2")) +c.close() + diff --git a/scripts/test_rancher_api.py b/scripts/test_rancher_api.py new file mode 100644 index 0000000..4c409b3 --- /dev/null +++ b/scripts/test_rancher_api.py @@ -0,0 +1,19 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +# Let's test Rancher endpoints +urls = [ + "https://aerodrome.snarfnet.net/v3/clusters", + "https://140.44.4.34/v3/clusters", + "http://140.44.4.11:80/v3/clusters" +] + +for u in urls: + try: + r = requests.get(u, verify=False, timeout=5) + print(u, "->", r.status_code, r.text[:200]) + except Exception as e: + print(u, "-> ERR:", e) + diff --git a/scripts/test_rancher_customapi.py b/scripts/test_rancher_customapi.py new file mode 100644 index 0000000..f5210b8 --- /dev/null +++ b/scripts/test_rancher_customapi.py @@ -0,0 +1,18 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +full_key = "kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7" +headers = { + "Authorization": f"Bearer {full_key}", + "Accept": "application/json" +} + +for cid in ["local", "c-cclnt"]: + r = requests.get(f"https://aerodrome.snarfnet.net/v3/clusters/{cid}", headers=headers, verify=False, timeout=10) + print(f"Cluster {cid} status: {r.status_code}") + if r.status_code == 200: + d = r.json() + print(f"Name: {d.get('name')}, State: {d.get('state')}") + print("Capacity:", d.get('capacity')) diff --git a/scripts/test_rancher_token.py b/scripts/test_rancher_token.py new file mode 100644 index 0000000..2640d7e --- /dev/null +++ b/scripts/test_rancher_token.py @@ -0,0 +1,30 @@ +import paramiko +import requests +import json +import urllib3 +urllib3.disable_warnings() + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +name = 'kubeconfig-u-aqzkufonck4hmbd' +_, o, _ = c.exec_command(f'kubectl get tokens.management.cattle.io {name} -o json') +data = json.loads(o.read().decode()) +token = data.get('token') + +full_key = f"{name}:{token}" +headers = {"Authorization": f"Bearer {full_key}"} +r = requests.get("https://aerodrome.snarfnet.net/v3/clusters", headers=headers, verify=False, timeout=10) + +for cl in r.json().get('data', []): + name = cl.get('name') + cid = cl.get('id') + state = cl.get('state') + cap = cl.get('capacity', {}) + alloc = cl.get('allocatable', {}) + print(f"=== Cluster: {name} (ID: {cid}, State: {state}) ===") + print(" Capacity:", cap) + print(" Allocatable:", alloc) + +c.close() diff --git a/scripts/test_service_widgets_all.py b/scripts/test_service_widgets_all.py new file mode 100644 index 0000000..36e24d0 --- /dev/null +++ b/scripts/test_service_widgets_all.py @@ -0,0 +1,23 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +headers = {"User-Agent": "Mozilla/5.0"} +base = "https://homepage.snarfnet.net/api/services/proxy" + +services = [ + ("GitOps & Automation", "Gitea", "repos"), + ("GitOps & Automation", "ArgoCD", "applications"), + ("Infrastructure & Observability", "Uptime Kuma", "status"), + ("Infrastructure & Observability", "Grafana", "stats"), + ("Environment & Weather", "Tempest Weather Station", "states"), +] + +for group, svc, ep in services: + try: + url = f"{base}?group={requests.utils.quote(group)}&service={requests.utils.quote(svc)}&index=0&endpoint={ep}" + r = requests.get(url, headers=headers, verify=False, timeout=10) + print(f"[{group} -> {svc}] status: {r.status_code}, len: {len(r.text)}") + except Exception as e: + print(f"[{group} -> {svc}] Error: {e}") + diff --git a/scripts/test_tempest_entities.py b/scripts/test_tempest_entities.py new file mode 100644 index 0000000..0870041 --- /dev/null +++ b/scripts/test_tempest_entities.py @@ -0,0 +1,20 @@ +import requests + +headers = { + "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4" +} + +sensors = [ + "sensor.st_00158555_temperature", + "sensor.st_00158555_feels_like", + "sensor.st_00158555_humidity", + "sensor.st_00158555_wind_speed" +] + +for s in sensors: + try: + r = requests.get(f"http://14.4.4.20:8123/api/states/{s}", headers=headers, timeout=5) + print(s, "->", r.status_code, r.json().get("state"), r.json().get("attributes", {}).get("unit_of_measurement")) + except Exception as e: + print(s, "Error:", e) + diff --git a/scripts/test_urls_quick.py b/scripts/test_urls_quick.py new file mode 100644 index 0000000..0d0ccba --- /dev/null +++ b/scripts/test_urls_quick.py @@ -0,0 +1,11 @@ +import requests +import urllib3 +urllib3.disable_warnings() + +for url in ["https://harvester.snarfnet.net", "https://jenkins.snarfnet.net", "http://140.44.4.70:8050"]: + try: + r = requests.get(url, verify=False, timeout=5) + print(f"{url} -> HTTP {r.status_code}") + except Exception as e: + print(f"{url} -> ERROR: {e}") + diff --git a/scripts/update_grafana_manifest.py b/scripts/update_grafana_manifest.py new file mode 100644 index 0000000..efb85ae --- /dev/null +++ b/scripts/update_grafana_manifest.py @@ -0,0 +1,97 @@ +import yaml +import json + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r", encoding="utf-8") as f: + docs = list(yaml.safe_load_all(f.read())) + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/systems-dashboard.json", "r", encoding="utf-8") as f: + systems_json = f.read() + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/container-metrics-dashboard.json", "r", encoding="utf-8") as f: + containers_json = f.read() + +# Deployment is doc 0 +deploy = docs[0] +pod_spec = deploy['spec']['template']['spec'] +container = pod_spec['containers'][0] + +# Add volume mounts if not present +mount_names = [m['name'] for m in container['volumeMounts']] +if 'systems-dashboard' not in mount_names: + container['volumeMounts'].append({ + 'mountPath': '/var/lib/grafana-dashboards/systems-dashboard.json', + 'name': 'systems-dashboard', + 'subPath': 'systems-dashboard.json' + }) + +if 'container-metrics-dashboard' not in mount_names: + container['volumeMounts'].append({ + 'mountPath': '/var/lib/grafana-dashboards/container-metrics-dashboard.json', + 'name': 'container-metrics-dashboard', + 'subPath': 'container-metrics-dashboard.json' + }) + +vol_names = [v['name'] for v in pod_spec['volumes']] +if 'systems-dashboard' not in vol_names: + pod_spec['volumes'].append({ + 'configMap': {'name': 'grafana-dashboard-systems'}, + 'name': 'systems-dashboard' + }) + +if 'container-metrics-dashboard' not in vol_names: + pod_spec['volumes'].append({ + 'configMap': {'name': 'grafana-dashboard-containers'}, + 'name': 'container-metrics-dashboard' + }) + +# Remove existing configmaps with these names if present in docs to update them +docs = [d for d in docs if not (d and d.get('kind') == 'ConfigMap' and d.get('metadata', {}).get('name') in ['grafana-dashboard-systems', 'grafana-dashboard-containers'])] + +cm_systems = { + 'apiVersion': 'v1', + 'kind': 'ConfigMap', + 'metadata': { + 'name': 'grafana-dashboard-systems', + 'namespace': 'default', + 'labels': { + 'app.kubernetes.io/name': 'grafana', + 'app.kubernetes.io/instance': 'grafana' + } + }, + 'data': { + 'systems-dashboard.json': systems_json + } +} + +cm_containers = { + 'apiVersion': 'v1', + 'kind': 'ConfigMap', + 'metadata': { + 'name': 'grafana-dashboard-containers', + 'namespace': 'default', + 'labels': { + 'app.kubernetes.io/name': 'grafana', + 'app.kubernetes.io/instance': 'grafana' + } + }, + 'data': { + 'container-metrics-dashboard.json': containers_json + } +} + +docs.append(cm_systems) +docs.append(cm_containers) + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "w", encoding="utf-8") as f: + yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +print("Updated manifests/aerodrome/grafana/grafana.yaml successfully!") + diff --git a/scripts/update_grafana_sa.py b/scripts/update_grafana_sa.py new file mode 100644 index 0000000..6d306d0 --- /dev/null +++ b/scripts/update_grafana_sa.py @@ -0,0 +1,14 @@ +import requests +import json +import urllib3 +urllib3.disable_warnings() + +auth = ('admin', 'Katzir476!!!') +sa_id = 3 +r = requests.patch(f"https://grafana.snarfnet.net/api/serviceaccounts/{sa_id}", auth=auth, json={"role": "Admin"}, verify=False) +print("Update role:", r.status_code, r.text) + +headers = {"Authorization": "Bearer glsa_71Y8I8DSZt3nXUNFzygnmBcEF4FUmWdw_1f03e67c"} +r_stats = requests.get("https://grafana.snarfnet.net/api/admin/stats", headers=headers, verify=False) +print("Admin stats:", r_stats.status_code, r_stats.text) + diff --git a/scripts/update_grafana_yaml.py b/scripts/update_grafana_yaml.py new file mode 100644 index 0000000..c56c687 --- /dev/null +++ b/scripts/update_grafana_yaml.py @@ -0,0 +1,129 @@ +import json + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/ai-stack-dashboard.json") as f: + dashboard_json = f.read() + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml") as f: + content = f.read() + +# Replace volumeMounts +old_mounts = """ volumeMounts: + - mountPath: /etc/grafana/grafana.ini + name: config + subPath: grafana.ini + - mountPath: /var/lib/grafana + name: storage""" + +new_mounts = """ volumeMounts: + - mountPath: /etc/grafana/grafana.ini + name: config + subPath: grafana.ini + - mountPath: /var/lib/grafana + name: storage + - mountPath: /etc/grafana/provisioning/datasources/datasources.yaml + name: datasources-config + subPath: datasources.yaml + - mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml + name: dashboards-provider-config + subPath: dashboards.yaml + - mountPath: /var/lib/grafana-dashboards/ai-stack-dashboard.json + name: ai-stack-dashboard + subPath: ai-stack-dashboard.json""" + +assert old_mounts in content, "old_mounts not found" +content = content.replace(old_mounts, new_mounts) + +# Replace volumes +old_vols = """ volumes: + - configMap: + defaultMode: 420 + name: grafana + name: config + - name: storage + persistentVolumeClaim: + claimName: grafana""" + +new_vols = """ volumes: + - configMap: + defaultMode: 420 + name: grafana + name: config + - name: storage + persistentVolumeClaim: + claimName: grafana + - configMap: + defaultMode: 420 + name: grafana-datasources + name: datasources-config + - configMap: + defaultMode: 420 + name: grafana-dashboards-provider + name: dashboards-provider-config + - configMap: + defaultMode: 420 + name: grafana-dashboard-ai-stack + name: ai-stack-dashboard""" + +assert old_vols in content, "old_vols not found" +content = content.replace(old_vols, new_vols) + +# Append ConfigMaps at the bottom +extra_cms = f"""--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: default + labels: + app.kubernetes.io/name: grafana +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-provider + namespace: default + labels: + app.kubernetes.io/name: grafana +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: 'ai-stack-provider' + orgId: 1 + folder: 'AI Stack & Inference' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana-dashboards +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-ai-stack + namespace: default + labels: + app.kubernetes.io/name: grafana +data: + ai-stack-dashboard.json: | +{chr(10).join(' ' + line for line in dashboard_json.splitlines())} +""" + +content = content.rstrip() + "\n" + extra_cms + +with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "w") as f: + f.write(content) + +print("Updated grafana.yaml successfully!") + diff --git a/scripts/update_homepage_bookmarks.py b/scripts/update_homepage_bookmarks.py new file mode 100644 index 0000000..5ea16c2 --- /dev/null +++ b/scripts/update_homepage_bookmarks.py @@ -0,0 +1,45 @@ +import yaml + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f: + text = f.read() + +docs = list(yaml.safe_load_all(text)) +cm = docs[2] + +bookmarks_yaml = """- Quick Links: + - GitHub: + - abbr: GH + href: https://github.com/ + - UniFi Site Manager: + - abbr: UI + href: https://unifi.ui.com/ + - Cloudflare: + - abbr: CF + href: https://dash.cloudflare.com/ +- Grafana Telemetry: + - Systems Performance: + - abbr: SYS + href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861 + - Container Workloads: + - abbr: K8S + href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads + - AI Stack Intelligence: + - abbr: AI + href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence +""" + +cm['data']['bookmarks.yaml'] = bookmarks_yaml + +with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f: + yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +print("Updated bookmarks in homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml successfully!") + diff --git a/scripts/update_homepage_manifest.py b/scripts/update_homepage_manifest.py new file mode 100644 index 0000000..b4fdced --- /dev/null +++ b/scripts/update_homepage_manifest.py @@ -0,0 +1,194 @@ +import os + +new_configmap_data = """apiVersion: v1 +data: + bookmarks.yaml: | + - Developer: + - Github: + - abbr: GH + href: https://github.com/ + docker.yaml: '' + kubernetes.yaml: | + mode: disable + services.yaml: | + - Critical Services: + - Authentik: + icon: authentik.png + href: https://authentik.snarfnet.net + widget: + type: authentik + url: https://authentik.snarfnet.net + fields: ["users", "loginsLast24H", "failedLoginsLast24H"] + key: 7HamapUhYizcNudHDIT5IFy3XEpRgeKgqr1Pw4ub9eTpAjMAzB8Aur1Z6dUU + - PeaNUT UPS: + icon: peanut.png + href: https://portal.snarfnet.net + widget: + type: peanut + url: https://peanut.snarfnet.net + fields: ["battery_charge", "ups_load", "ups_status"] + key: server-closet + - Unifi Controller: + icon: unifi.png + href: https://unifi.ui.com + widget: + type: unifi + url: https://140.44.44.1 + username: Snarf + password: Katzir476!!! + + - GitOps & Automation: + - Gitea: + icon: gitea.png + href: https://git.snarfnet.net + widget: + type: gitea + url: https://git.snarfnet.net + fields: ["repositories", "notifications", "issues", "pulls"] + key: 47c10c54574131c007302b246352f653facb5062 + - ArgoCD: + icon: argocd.png + href: https://argo.snarfnet.net + - AWX Automation: + icon: awx.png + href: https://awx.snarfnet.net + - Semaphore: + icon: semaphore.png + href: https://semaphore.snarfnet.net + + - Storage & Clusters: + - Longhorn Aerodrome: + icon: longhorn.png + href: https://longhorn.aerodrome.snarfnet.net + widget: + type: longhorn + url: https://longhorn.aerodrome.snarfnet.net + - Longhorn Spaceport: + icon: longhorn.png + href: https://longhorn.spaceport.snarfnet.net + widget: + type: longhorn + url: https://longhorn.spaceport.snarfnet.net + - Rancher (Aerodrome): + icon: rancher.png + href: https://aerodrome.snarfnet.net + widget: + type: rancher + url: https://aerodrome.snarfnet.net + cluster: local + - Harvester HCI (Spacedock): + icon: harvester.png + href: https://harvester.spacedock.snarfnet.net + - Portainer Swarm: + icon: portainer.png + href: https://shipyard.snarfnet.net + widget: + type: portainer + url: https://shipyard.snarfnet.net + env: 1 + + - Infrastructure & Observability: + - NGINX Proxy Manager: + icon: nginx-proxy-manager + href: https://nginx.snarfnet.net + widget: + type: npm + url: https://nginx.snarfnet.net + fields: ["enabled", "disabled", "total"] + username: snarf@snarfnet.net + password: Katzir476! + - Uptime Kuma: + icon: uptime-kuma + href: https://uptime.snarfnet.net + widget: + type: uptimekuma + url: https://uptime.snarfnet.net + fields: ["up", "down", "uptime", "incident"] + slug: public + - Zabbix: + icon: zabbix.png + href: https://zabbix.snarfnet.net + - Grafana: + icon: grafana.png + href: https://grafana.snarfnet.net + - Core Node Glances: + icon: glances.png + href: http://140.44.4.11:61208 + widget: + type: glances + url: http://140.44.4.11:61208 + version: 4 + metric: cpu + + - Environment & Weather: + - Tempest Weather Station: + icon: weather.png + href: https://snarfhome.snarfnet.net + description: Backyard Weather Telemetry + widget: + type: homeassistant + url: http://14.4.4.20:8123 + key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4 + custom: + - state: sensor.tempest_st_temperature + label: Outdoor Temp + - state: sensor.tempest_st_wind_speed + label: Wind Speed + - state: sensor.tempest_st_humidity + label: Humidity + - state: sensor.tempest_st_rain_rate + label: Rain Rate + + settings.yaml: | + title: SnarfNet Operations & Telemetry + headerStyle: boxedWidgets + color: green + theme: dark + layout: + Critical Services: + GitOps & Automation: + Storage & Clusters: + Infrastructure & Observability: + Environment & Weather: + showStats: true + providers: + longhorn: + url: https://longhorn.aerodrome.snarfnet.net + + widgets.yaml: | + - resources: + backend: resources + cpu: true + expanded: true + memory: true + - search: + provider: duckduckgo + target: _blank + - longhorn: + expanded: true + total: true + labels: true + nodes: true +""" + +filepath = r"homelab-gitops-repo\manifests\spaceport\homepage\homepage.yaml" +with open(filepath, "r", encoding="utf-8") as f: + text = f.read() + +# The configmap starts after the second "---" delimiter +parts = text.split("---\n") +if len(parts) == 1: + parts = text.split("---\r\n") + +# parts[0] is Deployment, parts[1] is Service, parts[2] is ConfigMap +cm_part = parts[2] +cm_kind_idx = cm_part.index("kind: ConfigMap") +cm_suffix = cm_part[cm_kind_idx:] + +new_cm_part = new_configmap_data + cm_suffix +parts[2] = new_cm_part + +new_full = "---\n".join(parts) +with open(filepath, "w", encoding="utf-8", newline="\r\n") as f: + f.write(new_full) +print("Updated homepage.yaml cleanly!") diff --git a/scripts/update_prom_manifest.py b/scripts/update_prom_manifest.py new file mode 100644 index 0000000..53ff58c --- /dev/null +++ b/scripts/update_prom_manifest.py @@ -0,0 +1,147 @@ +import yaml + +def str_presenter(dumper, data): + if '\n' in data: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f: + docs = list(yaml.safe_load_all(f.read())) + +# Doc 0: PVC +# Doc 1: ConfigMap +cm = docs[1] +prom_cfg = yaml.safe_load(cm['data']['prometheus.yml']) + +scrape_configs = prom_cfg.get('scrape_configs', []) + +# Check if node-exporter already exists +if not any(j.get('job_name') == 'node-exporter' for j in scrape_configs): + scrape_configs.append({ + 'job_name': 'node-exporter', + 'metrics_path': '/metrics', + 'static_configs': [ + { + 'targets': [ + '140.44.4.11:9100', '140.44.4.12:9100', '140.44.4.13:9100', + '140.44.4.21:9100', '140.44.4.22:9100', '140.44.4.23:9100', + '140.44.4.24:9100', '140.44.4.25:9100', '140.44.4.26:9100', + '140.44.4.27:9100' + ], + 'labels': { + 'cluster': 'aerodrome' + } + }, + { + 'targets': [ + '140.44.4.41:9100', '140.44.4.42:9100', '140.44.4.43:9100', + '140.44.4.51:9100', '140.44.4.52:9100', '140.44.4.53:9100', + '140.44.4.54:9100', '140.44.4.55:9100', '140.44.4.56:9100', + '140.44.4.57:9100' + ], + 'labels': { + 'cluster': 'spaceport' + } + } + ] + }) + +# Check if kubernetes-cadvisor already exists +if not any(j.get('job_name') == 'kubernetes-cadvisor' for j in scrape_configs): + scrape_configs.append({ + 'job_name': 'kubernetes-cadvisor', + 'scheme': 'https', + 'tls_config': { + 'ca_file': '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' + }, + 'bearer_token_file': '/var/run/secrets/kubernetes.io/serviceaccount/token', + 'kubernetes_sd_configs': [ + {'role': 'node'} + ], + 'relabel_configs': [ + { + 'action': 'labelmap', + 'regex': '__meta_kubernetes_node_label_(.+)' + }, + { + 'target_label': '__address__', + 'replacement': 'kubernetes.default.svc:443' + }, + { + 'source_labels': ['__meta_kubernetes_node_name'], + 'regex': '(.+)', + 'target_label': '__metrics_path__', + 'replacement': '/api/v1/nodes/${1}/proxy/metrics/cadvisor' + } + ] + }) + +prom_cfg['scrape_configs'] = scrape_configs +cm['data']['prometheus.yml'] = yaml.dump(prom_cfg, sort_keys=False, default_flow_style=False) + +# Doc 2: Deployment +deploy = docs[2] +deploy['spec']['template']['spec']['serviceAccountName'] = 'prometheus' + +# Add SA, ClusterRole, ClusterRoleBinding if not in docs +sa = { + 'apiVersion': 'v1', + 'kind': 'ServiceAccount', + 'metadata': { + 'name': 'prometheus', + 'namespace': 'default', + 'labels': {'app.kubernetes.io/name': 'prometheus'} + } +} + +cr = { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'ClusterRole', + 'metadata': { + 'name': 'prometheus', + 'labels': {'app.kubernetes.io/name': 'prometheus'} + }, + 'rules': [ + { + 'apiGroups': [''], + 'resources': ['nodes', 'nodes/proxy', 'nodes/metrics', 'services', 'endpoints', 'pods'], + 'verbs': ['get', 'list', 'watch'] + }, + { + 'nonResourceURLs': ['/metrics', '/metrics/cadvisor'], + 'verbs': ['get'] + } + ] +} + +crb = { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'ClusterRoleBinding', + 'metadata': { + 'name': 'prometheus', + 'labels': {'app.kubernetes.io/name': 'prometheus'} + }, + 'roleRef': { + 'apiGroup': 'rbac.authorization.k8s.io', + 'kind': 'ClusterRole', + 'name': 'prometheus' + }, + 'subjects': [ + { + 'kind': 'ServiceAccount', + 'name': 'prometheus', + 'namespace': 'default' + } + ] +} + +new_docs = [sa, cr, crb] + docs + +with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "w", encoding="utf-8") as f: + yaml.dump_all(new_docs, f, sort_keys=False, default_flow_style=False) + +print("Updated manifests/aerodrome/prometheus/prometheus.yaml successfully!") + diff --git a/scripts/update_shepherd_env.py b/scripts/update_shepherd_env.py new file mode 100644 index 0000000..d5f8d5d --- /dev/null +++ b/scripts/update_shepherd_env.py @@ -0,0 +1,45 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +shepherd_compose = '''version: "3.8" + +services: + shepherd: + image: containrrr/shepherd:latest + environment: + - SLEEP_TIME=24h + - IGNORELIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup + - UPDATE_OPTIONS=--with-registry-auth + - ROLLBACK_FAILURE=true + - TZ=America/New_York + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - node.role == manager + restart_policy: + condition: on-failure +''' + +sftp = c.open_sftp() +with sftp.file('/tmp/shepherd_compose.yml', 'w') as f: + f.write(shepherd_compose) +sftp.close() + +_, o, _ = c.exec_command("docker stack deploy -c /tmp/shepherd_compose.yml shepherd") +print(o.read().decode()) +time.sleep(3) +_, o, _ = c.exec_command("docker service ps shepherd_shepherd") +print(o.read().decode()) + +c.close() + diff --git a/scripts/upload_topology_to_homepage.py b/scripts/upload_topology_to_homepage.py new file mode 100644 index 0000000..9c6fda8 --- /dev/null +++ b/scripts/upload_topology_to_homepage.py @@ -0,0 +1,43 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + out = o.read().decode().strip() + err = e.read().decode().strip() + if err: + print('[stderr]', err) + return out + +with open('docs/snarfnet_homelab_topology.html', 'r', encoding='utf-8') as f: + html_content = f.read() + +with open('docs/snarfnet_homelab_topology.png', 'rb') as f: + png_content = f.read() + +# Send html +stdin, stdout, stderr = c.exec_command('cat > /tmp/topology.html') +stdin.write(html_content) +stdin.close() +stdout.read() + +# Send png +stdin, stdout, stderr = c.exec_command('cat > /tmp/topology.png') +stdin.write(png_content) +stdin.close() +stdout.read() + +pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'") +print(f'Target Pod: {pod}') + +print(run(f'kubectl cp /tmp/topology.png infrastructure/{pod}:/app/public/topology.png')) +print(run(f'kubectl cp /tmp/topology.html infrastructure/{pod}:/app/public/topology.html')) + +print('Files in /app/public:') +print(run(f'kubectl exec -n infrastructure {pod} -- ls -lh /app/public/topology*')) + +c.close() +print('Topology updated on Homepage successfully!') diff --git a/scripts/verify_grafana_dashboard.py b/scripts/verify_grafana_dashboard.py new file mode 100644 index 0000000..8e1a4b5 --- /dev/null +++ b/scripts/verify_grafana_dashboard.py @@ -0,0 +1,36 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, e = c.exec_command(cmd) + return o.read().decode().strip() + +pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'") +print(run(f"kubectl exec {pod} -- cat /var/lib/grafana-dashboards/ai-stack-dashboard.json | grep -i 'title' | head -n 5")) + +py_script = """import sqlite3 +import subprocess +out = subprocess.check_output("kubectl exec deployment/grafana -- cat /var/lib/grafana/grafana.db", shell=True) +with open("/tmp/g.db", "wb") as f: + f.write(out) +conn = sqlite3.connect('/tmp/g.db') +cur = conn.cursor() +print('=== DATASOURCES ===') +for row in cur.execute('SELECT id, name, type, url FROM data_source').fetchall(): + print(row) +print('=== DASHBOARDS ===') +for row in cur.execute('SELECT id, title, slug, uid FROM dashboard').fetchall(): + print(row) +""" + +stdin, stdout, stderr = c.exec_command("cat > /tmp/check_g.py && python3 /tmp/check_g.py") +stdin.write(py_script) +stdin.close() +print(stdout.read().decode()) +print(stderr.read().decode()) + +c.close() + diff --git a/scripts/verify_heimdall_clean.py b/scripts/verify_heimdall_clean.py new file mode 100644 index 0000000..605f689 --- /dev/null +++ b/scripts/verify_heimdall_clean.py @@ -0,0 +1,29 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +script = ''' +import sqlite3 +con = sqlite3.connect('/var/lib/docker-data/heimdall/config/www/app.sqlite') +cur = con.cursor() +print("Total items:", cur.execute("SELECT count(*) FROM items").fetchone()[0]) +print("Duplicates:", cur.execute("SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1").fetchall()) +print("Soft-deleted items:", cur.execute("SELECT count(*) FROM items WHERE deleted_at IS NOT NULL").fetchone()[0]) +print("\\nActive items (ordered):") +for r in cur.execute("SELECT id, title, [order], pinned FROM items WHERE id != 0 ORDER BY [order]").fetchall(): + print(" #%-2d (id: %-2d, pinned: %d): %s" % (r[2], r[0], r[3], r[1])) +''' + +sftp = c.open_sftp() +with sftp.file('/tmp/verify_heimdall.py', 'w') as f: + f.write(script) +sftp.close() + +_, stdout, _ = c.exec_command('python3 /tmp/verify_heimdall.py') +print(stdout.read().decode()) +c.close() + diff --git a/scripts/verify_kuma_live.py b/scripts/verify_kuma_live.py new file mode 100644 index 0000000..cee17d6 --- /dev/null +++ b/scripts/verify_kuma_live.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n>>> {cmd}") + _, o, e = c.exec_command(cmd) + res = o.read().decode().strip() + if res: + print(res) + return res + +pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'") +run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT count(*) AS total_monitors FROM monitor; SELECT name, type, active, is_default FROM notification; SELECT count(*) AS linked_notifications FROM monitor_notification;" """) +run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, url FROM monitor ORDER BY id DESC LIMIT 10;" """) + +c.close() + diff --git a/scripts/verify_node_metrics.py b/scripts/verify_node_metrics.py new file mode 100644 index 0000000..139b092 --- /dev/null +++ b/scripts/verify_node_metrics.py @@ -0,0 +1,16 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + _, o, _ = c.exec_command(cmd) + return o.read().decode().strip() + +print("Node load on 140.44.4.11:") +print(run("curl -s http://localhost:9100/metrics | grep -E '^node_load1 '")) +print("Node memory total:") +print(run("curl -s http://localhost:9100/metrics | grep -E '^node_memory_MemTotal_bytes '")) +c.close() + diff --git a/snarfnet-game-seeds/README.md b/snarfnet-game-seeds/README.md new file mode 100644 index 0000000..bd73063 --- /dev/null +++ b/snarfnet-game-seeds/README.md @@ -0,0 +1,73 @@ +# šŸŒ² SnarfNet Game Seed Management Plane + +Declarative, GitOps-ready game server management plane for the SnarfNet dedicated gaming machine (`bc:24:11:67:16:cd` @ `140.44.44.80`), modeled on the declarative seed pattern (`arasoi/yggdrasil-releases` and `arasoi/yggdrasil-seeds`). + +--- + +## šŸŽ® Declared Game Servers + +| Game | Steam AppID | Container Image | Ports | RAM (Res / Limit) | World Save Path | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **Enshrouded** | `2278520` | `skarab42/enshrouded-server:latest` | `15636/udp`, `15637/udp` | 8GB / 16GB | `/opt/game-servers/enshrouded/savegame` | +| **Farming Simulator 25** | `2300320` | `thijsvanloef/farming-simulator-25:latest` | `10823/tcp`, `10823/udp` | 6GB / 12GB | `/opt/game-servers/fs25/savegame` | +| **Stationeers** | `600760` | `sprits/stationeers-server:latest` | `27016/udp`, `27015/udp` | 4GB / 8GB | `/opt/game-servers/stationeers/saves` | +| **Valheim** (Crossplay) | `896660` | `lloesche/valheim-server:latest` | `2456/udp`, `2457/udp` | 4GB / 8GB | `/opt/game-servers/valheim/worlds` | +| **V Rising** | `1829350` | `trueosiris/vrising:latest` | `9876/udp`, `9877/udp` | 4GB / 8GB | `/opt/game-servers/vrising/Saves` | + +--- + +## šŸ› ļø Management CLI (`core/manager.py`) + +The manager CLI reconciles declared seeds against the host runtime, triggers automated world backups to Synology NAS, and issues Wake-on-LAN magic packets. + +### Commands + +```bash +# 1. Preview declared server configuration +python core/manager.py plan + +# 2. Render docker-compose.yml for live deployment +python core/manager.py compose + +# 3. Trigger manual atomic snapshot & sync to Synology NAS +python core/manager.py backup + +# 4. Wake up the gaming PC via Wake-on-LAN (bc:24:11:67:16:cd) +python core/manager.py wol +``` + +--- + +## šŸ’¾ Backup & Disaster Recovery Architecture + +All game world save files are preserved in atomic tarballs and synchronized to the Synology NAS: +- **NAS Backup NFS Target**: `14.10.10.71:/volume1/docker/game-backups` +- **Host Mount**: `/mnt/nas/game-backups` +- **Automated Timer**: Daily at 04:00 AM via `gameseed-backup.timer` +- **Retention**: 14 rolling snapshots per game + +--- + +## šŸŒ UDM Pro Port Forwarding Matrix + +To permit external friends and players to join the servers, ensure the following port forwards are mapped from WAN to `140.44.44.80` on the UDM Pro: + +``` +Enshrouded: UDP 15636-15637 -> 140.44.44.80 +Farming Simulator 25: TCP/UDP 10823 -> 140.44.44.80 +Stationeers: UDP 27015-27016 -> 140.44.44.80 +Valheim: UDP 2456-2457 -> 140.44.44.80 +V Rising: UDP 9876-9877 -> 140.44.44.80 +``` + +--- + +## šŸš€ Bootstrap New Host + +To provision the physical gaming PC: +```bash +sudo bash scripts/setup-host.sh +python3 core/manager.py compose +docker compose up -d +``` + diff --git a/snarfnet-game-seeds/core/manager.py b/snarfnet-game-seeds/core/manager.py new file mode 100644 index 0000000..2618f1f --- /dev/null +++ b/snarfnet-game-seeds/core/manager.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +""" +SnarfNet Game Seed Management Plane (Yggdrasil-compatible) +Declarative orchestrator for private game servers on SnarfNet Gaming PC. +""" + +import os +import sys +import glob +import yaml +import socket +import struct +import tarfile +import datetime +import subprocess +from pathlib import Path + +TARGET_MAC = "bc:24:11:67:16:cd" +BROADCAST_IP = "140.44.44.255" +NAS_BACKUP_ROOT = "/mnt/nas/game-backups" +SEEDS_DIR = Path(__file__).parent.parent / "seeds" + +def send_wol(mac=TARGET_MAC, bcast=BROADCAST_IP): + """Sends a Wake-on-LAN magic packet to the gaming PC.""" + clean_mac = mac.replace(":", "").replace("-", "") + if len(clean_mac) != 12: + raise ValueError(f"Invalid MAC address: {mac}") + data = bytes.fromhex("FF" * 6 + clean_mac * 16) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + s.sendto(data, (bcast, 9)) + print(f"[WOL] Sent magic packet to {mac} via {bcast}:9") + +def load_seeds(): + """Loads all declarative seed YAML files from the seeds directory.""" + seeds = [] + for filepath in glob.glob(str(SEEDS_DIR / "*.seed.yaml")): + with open(filepath, "r", encoding="utf-8") as f: + docs = yaml.safe_load(f) + if docs and docs.get("kind") == "GameSeed": + seeds.append(docs) + return seeds + +def cmd_plan(): + """Previews desired state across all defined game seeds.""" + seeds = load_seeds() + print(f"\n=======================================================") + print(f"šŸŒ² SnarfNet Game Seed Plan ({len(seeds)} Seeds Declared)") + print(f"=======================================================") + for s in seeds: + meta = s.get("metadata", {}) + spec = s.get("spec", {}) + game = spec.get("game", {}) + server = spec.get("server", {}) + net = spec.get("network", {}) + runtime = spec.get("runtime", {}) + res = spec.get("resources", {}) + + ports = ", ".join([f"{p['port']}/{p['protocol']}" for p in net.get("ports", [])]) + print(f"\nšŸŽ® [{meta.get('name')}] {meta.get('displayName')}") + print(f" Steam AppID: {game.get('appId')} (Branch: {game.get('branch')})") + print(f" Image: {runtime.get('image')}") + print(f" Ports: {ports}") + print(f" Memory: {res.get('memoryReservation')} -> {res.get('memoryLimit')}") + print(f" Save Dir: {server.get('saveDirectory')}") + print(f" Backup: {spec.get('backup', {}).get('schedule')} (Retention: {spec.get('backup', {}).get('retentionDays')}d)") + +def cmd_generate_compose(output_path="docker-compose.yml"): + """Generates standard docker-compose.yml from all declared seeds.""" + seeds = load_seeds() + services = {} + + for s in seeds: + name = s["metadata"]["name"] + spec = s["spec"] + runtime = spec.get("runtime", {}) + server = spec.get("server", {}) + net = spec.get("network", {}) + res = spec.get("resources", {}) + + ports = [] + for p in net.get("ports", []): + ports.append(f"{p['port']}:{p['port']}/{p['protocol'].lower()}") + + save_dir = server.get("saveDirectory") + data_dir = f"/opt/game-servers/{name}/data" + + services[name] = { + "image": runtime.get("image"), + "container_name": f"gameseed-{name}", + "restart": runtime.get("restartPolicy", "unless-stopped"), + "environment": runtime.get("environment", {}), + "ports": ports, + "volumes": [ + f"{save_dir}:{save_dir}", + f"{data_dir}:/serverdata" + ], + "deploy": { + "resources": { + "limits": { + "memory": res.get("memoryLimit", "8Gi"), + "cpus": res.get("cpuLimit", "4") + }, + "reservations": { + "memory": res.get("memoryReservation", "4Gi") + } + } + } + } + + compose = { + "version": "3.8", + "services": services + } + + with open(output_path, "w", encoding="utf-8") as f: + yaml.dump(compose, f, sort_keys=False) + print(f"[COMPOSE] Rendered {len(services)} services to {output_path}") + +def cmd_backup(): + """Executes atomic snapshots of all world save directories and copies to Synology NAS.""" + seeds = load_seeds() + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + nas_root = Path(NAS_BACKUP_ROOT) + + print(f"\nšŸ“¦ Starting Declarative Game Save Backups ({timestamp})...") + for s in seeds: + name = s["metadata"]["name"] + save_dir = Path(s["spec"]["server"]["saveDirectory"]) + target_dir = nas_root / name + target_dir.mkdir(parents=True, exist_ok=True) + + archive_name = f"{name}_save_{timestamp}.tar.gz" + archive_path = target_dir / archive_name + + if save_dir.exists(): + print(f" Archiving {name}: {save_dir} -> {archive_path}") + with tarfile.open(archive_path, "w:gz") as tar: + tar.add(save_dir, arcname=name) + print(f" āœ“ Backup complete: {archive_path} ({archive_path.stat().st_size} bytes)") + else: + print(f" [SKIP] Save directory {save_dir} not found on host.") + +def main(): + if len(sys.argv) < 2: + print("Usage: python manager.py ") + sys.exit(1) + + action = sys.argv[1].lower() + if action == "plan": + cmd_plan() + elif action == "compose": + cmd_generate_compose() + elif action == "backup": + cmd_backup() + elif action == "wol": + send_wol() + else: + print(f"Unknown command: {action}") + sys.exit(1) + +if __name__ == "__main__": + main() + diff --git a/snarfnet-game-seeds/scripts/setup-host.sh b/snarfnet-game-seeds/scripts/setup-host.sh new file mode 100644 index 0000000..befda88 --- /dev/null +++ b/snarfnet-game-seeds/scripts/setup-host.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# ============================================================================== +# SnarfNet Gaming PC Bootstrap Script +# MAC: bc:24:11:67:16:cd | IP: 140.44.44.80 (Compute / Game VLAN) +# ============================================================================== +set -euo pipefail + +echo "=======================================================" +echo "šŸŒ² Bootstrapping SnarfNet Dedicated Game Server Host" +echo "=======================================================" + +# 1. System Updates & Essential Tools +echo ">>> Installing prerequisites and system tools..." +apt-get update && apt-get install -y \ + curl wget git jq ufw nfs-common ethtool net-tools \ + python3 python3-yaml python3-pip apt-transport-https \ + ca-certificates gnupg lsb-release + +# 2. Configure Wake-on-LAN (Persistent via systemd) +echo ">>> Configuring Wake-on-LAN for primary interface..." +NIC=$(ip route show default | awk '{print $5}' | head -n1) +ethtool -s "$NIC" wol g || true + +cat < /etc/systemd/system/wol.service +[Unit] +Description=Enable Wake-on-LAN on $NIC +After=network.target + +[Service] +Type=oneshot +ExecStart=/sbin/ethtool -s $NIC wol g + +[Install] +WantedBy=basic.target +EOF +systemctl daemon-reload +systemctl enable wol.service + +# 3. Mount Synology NAS Game Backup Share +echo ">>> Configuring Synology NAS NFS Mount (14.10.10.71:/volume1/docker/game-backups)..." +mkdir -p /mnt/nas/game-backups +if ! grep -qs '/mnt/nas/game-backups' /etc/fstab; then + echo "14.10.10.71:/volume1/docker/game-backups /mnt/nas/game-backups nfs rw,hard,intr,nolock,timeo=14 0 0" >> /etc/fstab +fi +mount -a || true + +# 4. Install Docker & Compose (if not present) +if ! command -v docker &> /dev/null; then + echo ">>> Installing Docker Engine..." + curl -fsSL https://get.docker.com | sh + systemctl enable docker + systemctl start docker +fi + +# 5. Prepare Game Directory Structure +echo ">>> Creating game server storage paths..." +mkdir -p /opt/game-servers/{enshrouded,fs25,stationeers,valheim,vrising}/{savegame,saves,worlds,data} +chmod -R 775 /opt/game-servers + +# 6. Setup Seed Manager Service & Daily Cron +echo ">>> Installing SnarfNet Seed Reconciler and Backup Timer..." +cat < /etc/systemd/system/gameseed-backup.service +[Unit] +Description=SnarfNet Declarative Game Server Save Backup +After=network.target docker.service + +[Service] +Type=oneshot +WorkingDirectory=/opt/snarfnet-game-seeds +ExecStart=/usr/bin/python3 /opt/snarfnet-game-seeds/core/manager.py backup + +[Install] +WantedBy=multi-user.target +EOF + +cat < /etc/systemd/system/gameseed-backup.timer +[Unit] +Description=Nightly Game Save Backup Timer (04:00 AM) + +[Timer] +OnCalendar=*-*-* 04:00:00 +Persistent=true + +[Install] +Timers.target +EOF + +systemctl daemon-reload +systemctl enable gameseed-backup.timer +systemctl start gameseed-backup.timer + +echo "=======================================================" +echo "āœ“ Host bootstrap complete!" +echo "Run: python3 /opt/snarfnet-game-seeds/core/manager.py compose && docker compose up -d" +echo "=======================================================" + diff --git a/snarfnet-game-seeds/seeds/enshrouded.seed.yaml b/snarfnet-game-seeds/seeds/enshrouded.seed.yaml new file mode 100644 index 0000000..b1efebc --- /dev/null +++ b/snarfnet-game-seeds/seeds/enshrouded.seed.yaml @@ -0,0 +1,42 @@ +apiVersion: yggdrasil.snarfnet.net/v1alpha1 +kind: GameSeed +metadata: + name: enshrouded + displayName: "Enshrouded Dedicated Server" + version: "1.0.0" +spec: + game: + appId: 2278520 + branch: public + autoUpdate: true + server: + name: "SnarfNet Enshrouded Sanctuary" + passwordSecret: "ENSHROUDED_SERVER_PASSWORD" + maxPlayers: 16 + saveDirectory: "/opt/game-servers/enshrouded/savegame" + network: + ip: "140.44.44.80" + ports: + - port: 15636 + protocol: UDP + description: "Game Query & Traffic" + - port: 15637 + protocol: UDP + description: "Steam Query Port" + resources: + cpuLimit: "8" + memoryLimit: "16Gi" + memoryReservation: "8Gi" + runtime: + image: "skarab42/enshrouded-server:latest" + restartPolicy: unless-stopped + environment: + SERVER_NAME: "SnarfNet Enshrouded Sanctuary" + SERVER_SLOTS: 16 + GAME_PORT: 15636 + QUERY_PORT: 15637 + backup: + schedule: "0 4 * * *" # Daily 04:00 AM + retentionDays: 14 + targetPath: "/mnt/nas/game-backups/enshrouded" + diff --git a/snarfnet-game-seeds/seeds/farming-simulator-25.seed.yaml b/snarfnet-game-seeds/seeds/farming-simulator-25.seed.yaml new file mode 100644 index 0000000..ab9437f --- /dev/null +++ b/snarfnet-game-seeds/seeds/farming-simulator-25.seed.yaml @@ -0,0 +1,41 @@ +apiVersion: yggdrasil.snarfnet.net/v1alpha1 +kind: GameSeed +metadata: + name: farming-simulator-25 + displayName: "Farming Simulator 25 Dedicated Server" + version: "1.0.0" +spec: + game: + appId: 2300320 + branch: public + autoUpdate: true + server: + name: "SnarfNet FS25 Countryside" + passwordSecret: "FS25_SERVER_PASSWORD" + adminPasswordSecret: "FS25_ADMIN_PASSWORD" + maxPlayers: 16 + saveDirectory: "/opt/game-servers/fs25/savegame" + network: + ip: "140.44.44.80" + ports: + - port: 10823 + protocol: TCP + description: "Game Web Management & Client Connection" + - port: 10823 + protocol: UDP + description: "Game Network Traffic" + resources: + cpuLimit: "6" + memoryLimit: "12Gi" + memoryReservation: "6Gi" + runtime: + image: "thijsvanloef/farming-simulator-25:latest" + restartPolicy: unless-stopped + environment: + SERVER_NAME: "SnarfNet FS25 Countryside" + GAME_PORT: 10823 + backup: + schedule: "0 4 * * *" + retentionDays: 14 + targetPath: "/mnt/nas/game-backups/farming-simulator-25" + diff --git a/snarfnet-game-seeds/seeds/stationeers.seed.yaml b/snarfnet-game-seeds/seeds/stationeers.seed.yaml new file mode 100644 index 0000000..b0dcda6 --- /dev/null +++ b/snarfnet-game-seeds/seeds/stationeers.seed.yaml @@ -0,0 +1,44 @@ +apiVersion: yggdrasil.snarfnet.net/v1alpha1 +kind: GameSeed +metadata: + name: stationeers + displayName: "Stationeers Dedicated Server" + version: "1.0.0" +spec: + game: + appId: 600760 + branch: public + autoUpdate: true + server: + name: "SnarfNet Stationeers Moonbase" + worldName: "Moon" + passwordSecret: "STATIONEERS_SERVER_PASSWORD" + maxPlayers: 10 + saveDirectory: "/opt/game-servers/stationeers/saves" + network: + ip: "140.44.44.80" + ports: + - port: 27016 + protocol: UDP + description: "Game Traffic Port" + - port: 27015 + protocol: UDP + description: "Steam Query Port" + resources: + cpuLimit: "4" + memoryLimit: "8Gi" + memoryReservation: "4Gi" + runtime: + image: "sprits/stationeers-server:latest" + restartPolicy: unless-stopped + environment: + SERVER_NAME: "SnarfNet Stationeers Moonbase" + WORLD_NAME: "Moon" + GAME_PORT: 27016 + QUERY_PORT: 27015 + AUTO_SAVE_INTERVAL: 300 + backup: + schedule: "0 4 * * *" + retentionDays: 14 + targetPath: "/mnt/nas/game-backups/stationeers" + diff --git a/snarfnet-game-seeds/seeds/valheim.seed.yaml b/snarfnet-game-seeds/seeds/valheim.seed.yaml new file mode 100644 index 0000000..c42841b --- /dev/null +++ b/snarfnet-game-seeds/seeds/valheim.seed.yaml @@ -0,0 +1,47 @@ +apiVersion: yggdrasil.snarfnet.net/v1alpha1 +kind: GameSeed +metadata: + name: valheim + displayName: "Valheim Dedicated Server" + version: "1.0.0" +spec: + game: + appId: 896660 + branch: public + autoUpdate: true + server: + name: "SnarfNet Valheim Valhalla" + worldName: "Snarfheim" + passwordSecret: "VALHEIM_SERVER_PASSWORD" + crossplay: true + maxPlayers: 10 + saveDirectory: "/opt/game-servers/valheim/worlds" + network: + ip: "140.44.44.80" + ports: + - port: 2456 + protocol: UDP + description: "Game Traffic Port" + - port: 2457 + protocol: UDP + description: "Steam Query Port" + resources: + cpuLimit: "6" + memoryLimit: "8Gi" + memoryReservation: "4Gi" + runtime: + image: "lloesche/valheim-server:latest" + restartPolicy: unless-stopped + environment: + SERVER_NAME: "SnarfNet Valheim Valhalla" + WORLD_NAME: "Snarfheim" + SERVER_PORT: 2456 + SERVER_PUBLIC: 1 + CROSSPLAY: "true" + UPDATE_CRON: "0 5 * * *" + BEPINEX: "false" + backup: + schedule: "0 4 * * *" + retentionDays: 14 + targetPath: "/mnt/nas/game-backups/valheim" + diff --git a/snarfnet-game-seeds/seeds/vrising.seed.yaml b/snarfnet-game-seeds/seeds/vrising.seed.yaml new file mode 100644 index 0000000..7212022 --- /dev/null +++ b/snarfnet-game-seeds/seeds/vrising.seed.yaml @@ -0,0 +1,43 @@ +apiVersion: yggdrasil.snarfnet.net/v1alpha1 +kind: GameSeed +metadata: + name: vrising + displayName: "V Rising Dedicated Server" + version: "1.0.0" +spec: + game: + appId: 1829350 + branch: public + autoUpdate: true + server: + name: "SnarfNet V Rising Vampire Lair" + saveName: "SnarfNet_Vampires" + passwordSecret: "VRISING_SERVER_PASSWORD" + maxPlayers: 20 + saveDirectory: "/opt/game-servers/vrising/Saves" + network: + ip: "140.44.44.80" + ports: + - port: 9876 + protocol: UDP + description: "Game Traffic Port" + - port: 9877 + protocol: UDP + description: "Query Port" + resources: + cpuLimit: "6" + memoryLimit: "8Gi" + memoryReservation: "4Gi" + runtime: + image: "trueosiris/vrising:latest" + restartPolicy: unless-stopped + environment: + SERVERNAME: "SnarfNet V Rising Vampire Lair" + SAVENAME: "SnarfNet_Vampires" + GAMEPORT: 9876 + QUERYPORT: 9877 + backup: + schedule: "0 4 * * *" + retentionDays: 14 + targetPath: "/mnt/nas/game-backups/vrising" +