Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add homelab deployment configs, documentation, and automation scripts

Browse Source
This commit is contained in:
Snarf
2026-11-12 16:57:08 -05:00
parent 1228b957de
commit 5a70460e89
270 changed files with 9709 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
docker-deployments/heimdall.yml
View File

@@ -1,20 +1,33 @@
version: "3.8"
services:
heimdall:
image: lscr.io/linuxserver/heimdall:latest
ports:
- "85:80"
- "403:443"
- target: 80
published: 85
mode: ingress
- target: 443
published: 403
mode: ingress
environment:
- PUID=1000
- PGID=1000
- TZ=America/New_York
volumes:
- heimdall_config:/config
- type: bind
source: /var/lib/docker-data/heimdall/config
target: /config
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.hostname == snshpd4lnxswm01
restart_policy:
condition: on-failure
volumes:
heimdall_config:
driver: local
driver_opts:
type: nfs
o: "addr=14.10.10.71,rw,nfsvers=4"
device: ":/volume1/docker/heimdall/"
networks:
default:
name: homelab_public
external: true

21
docker-deployments/shepherd.yml Normal file
View File

@@ -0,0 +1,21 @@
version: "3.8"
services:
shepherd:
image: containrrr/shepherd:latest
environment:
- SLEEP_TIME=24h
- IGNORELIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup
- UPDATE_OPTIONS=--with-registry-auth
- ROLLBACK_FAILURE=true
- TZ=America/New_York
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure

1029
docs/snarfnet_homelab_topology.html Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
docs/snarfnet_homelab_topology.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 789 KiB

2
homelab-gitops-repo

Submodule homelab-gitops-repo updated: 7b2dd24526...c7eb47f6d8

58
scripts/add_kuma_bookmarks.py Normal file
View File

@@ -0,0 +1,58 @@
import yaml
def str_presenter(dumper, data):
if '\n' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
text = f.read()
docs = list(yaml.safe_load_all(text))
cm = docs[2]
bookmarks_yaml = """- Quick Links:
- GitHub:
- abbr: GH
href: https://github.com/
- UniFi Site Manager:
- abbr: UI
href: https://unifi.ui.com/
- Cloudflare:
- abbr: CF
href: https://dash.cloudflare.com/
- Grafana Telemetry:
- Systems Performance:
- abbr: SYS
href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861
- Container Workloads:
- abbr: K8S
href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads
- AI Stack Intelligence:
- abbr: AI
href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence
- Uptime Status Pages:
- Unified Overview:
- abbr: ALL
href: https://uptime.snarfnet.net/status/default
- Core & VIP Fabric:
- abbr: VIP
href: https://uptime.snarfnet.net/status/core
- GitOps & CI/CD:
- abbr: OPS
href: https://uptime.snarfnet.net/status/gitops
- Edge & Ingress:
- abbr: EDG
href: https://uptime.snarfnet.net/status/edge
"""
cm['data']['bookmarks.yaml'] = bookmarks_yaml
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
print("Updated bookmarks in homepage.yaml successfully!")

26
scripts/add_topology_to_homepage.py Normal file
View File

@@ -0,0 +1,26 @@
import yaml
def str_presenter(dumper, data):
if '\n' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
text = f.read()
docs = list(yaml.safe_load_all(text))
cm = docs[2]
services = yaml.safe_load(cm['data']['services.yaml'])
bookmarks = yaml.safe_load(cm['data']['bookmarks.yaml'])
cm['data']['services.yaml'] = yaml.dump(services, sort_keys=False, default_flow_style=False, allow_unicode=True)
cm['data']['bookmarks.yaml'] = yaml.dump(bookmarks, sort_keys=False, default_flow_style=False, allow_unicode=True)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
print("Cleaned up unicode strings in homepage.yaml!")

118
scripts/add_zabbix_devices.py Normal file
View File

@@ -0,0 +1,118 @@
import json
import ssl
import urllib.request
ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
ZABBIX_USER = "Admin"
ZABBIX_PASS = "zabbix"
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def zbx_call(method, params, auth=None):
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
headers = {"Content-Type": "application/json-rpc"}
if auth:
headers["Authorization"] = f"Bearer {auth}"
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
resp = json.loads(r.read().decode("utf-8"))
if "error" in resp:
raise RuntimeError(f"Zabbix error in {method}: {resp['error']}")
return resp["result"]
auth = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS})
# Check existing host names
existing_hosts = zbx_call("host.get", {"output": ["hostid", "host", "name"]}, auth)
existing_names = {h["host"].lower(): h["hostid"] for h in existing_hosts}
# Create host groups if not exist
groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth)
group_map = {g["name"]: g["groupid"] for g in groups}
for gname in ["Network Devices", "Environmental & Sensors", "Management & IPMI"]:
if gname not in group_map:
res = zbx_call("hostgroup.create", {"name": gname}, auth)
group_map[gname] = res["groupids"][0]
print(f"Created Host Group: {gname} (ID: {group_map[gname]})")
devices_to_add = [
{
"host": "idrac-spacedock4",
"name": "Dell PowerEdge R730 Node 4 iDRAC (idrac-spacedock4)",
"ip": "140.44.44.124",
"group": "Management & IPMI"
},
{
"host": "usw-agg-core",
"name": "UniFi Switch Aggregation 10G Core",
"ip": "14.44.4.26",
"group": "Network Devices"
},
{
"host": "usw-24-poe-closet",
"name": "UniFi Switch 24 PoE Closet",
"ip": "14.44.4.61",
"group": "Network Devices"
},
{
"host": "usw-pro-24-poe-compute",
"name": "UniFi Switch Pro 24 PoE Compute",
"ip": "14.44.4.107",
"group": "Network Devices"
},
{
"host": "usw-flex-attic",
"name": "UniFi Switch Flex Attic",
"ip": "14.44.4.98",
"group": "Network Devices"
},
{
"host": "usw-flex-garage",
"name": "UniFi Switch Flex Garage",
"ip": "14.44.4.96",
"group": "Network Devices"
},
{
"host": "tempest-weather-station",
"name": "Tempest Weather Station Gateway",
"ip": "14.4.4.47",
"group": "Environmental & Sensors"
},
{
"host": "sense-energy-monitor",
"name": "Sense Home Energy Gateway",
"ip": "14.4.4.48",
"group": "Environmental & Sensors"
}
]
for dev in devices_to_add:
hname = dev["host"]
if hname.lower() in existing_names:
print(f"[EXISTS] {hname}")
continue
params = {
"host": dev["host"],
"name": dev["name"],
"interfaces": [
{
"type": 1, # Agent / ICMP
"main": 1,
"useip": 1,
"ip": dev["ip"],
"dns": "",
"port": "10050"
}
],
"groups": [{"groupid": group_map[dev["group"]]}],
"templates": [{"templateid": "10564"}] # ICMP Ping
}
try:
res = zbx_call("host.create", params, auth)
print(f"[CREATED] {dev['name']} ({dev['ip']}) -> HostID {res['hostids'][0]}")
except Exception as e:
print(f"[FAILED] {dev['name']}: {e}")

250
scripts/apply_full_homepage.py Normal file
View File

@@ -0,0 +1,250 @@
import yaml
def str_presenter(dumper, data):
if '\n' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
text = f.read()
docs = list(yaml.safe_load_all(text))
cm = docs[2]
services_yaml = """- Critical Services:
- Authentik:
icon: authentik.png
href: https://authentik.snarfnet.net
description: SSO & Identity Management
- PeaNUT UPS:
icon: peanut.png
href: https://portal.snarfnet.net
description: UPS Power & Battery Monitor
- Unifi Controller:
icon: unifi.png
href: https://unifi.ui.com
widget:
type: unifi
url: https://140.44.44.1
username: Snarf
password: Katzir476!!!
- GitOps & Automation:
- Gitea:
icon: gitea.png
href: https://git.snarfnet.net
description: Git Repositories
widget:
type: gitea
url: https://git.snarfnet.net
key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71
- ArgoCD:
icon: argocd.png
href: https://argo.snarfnet.net
description: Declarative GitOps Reconciler
widget:
type: argocd
url: https://argo.snarfnet.net
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0
- AWX Automation:
icon: awx.png
href: https://awx.snarfnet.net
description: Ansible Automation Controller
- Semaphore:
icon: semaphore.png
href: https://semaphore.snarfnet.net
description: Playbook Execution Engine
- Storage & Clusters:
- Longhorn Aerodrome:
icon: longhorn.png
href: https://longhorn.aerodrome.snarfnet.net
description: Aerodrome Distributed Flash Storage
- Longhorn Spaceport:
icon: longhorn.png
href: https://longhorn.spaceport.snarfnet.net
description: Spaceport Distributed Block Storage
- Rancher (Aerodrome):
icon: rancher.png
href: https://aerodrome.snarfnet.net
description: Kubernetes Management Console
widget:
type: customapi
url: https://aerodrome.snarfnet.net/v3/clusters/local
headers:
Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
mappings:
- field: capacity.cpu
label: CPU Cores
- field: capacity.pods
label: Max Pods
- field: state
label: Cluster State
- Harvester HCI (Spacedock):
icon: harvester.png
href: https://harvester.spacedock.snarfnet.net
description: HCI Virtualization Hypervisors
widget:
type: customapi
url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt
headers:
Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
mappings:
- field: capacity.cpu
label: CPU Cores
- field: capacity.pods
label: Max Pods
- field: state
label: HCI State
- Portainer Swarm:
icon: portainer.png
href: https://shipyard.snarfnet.net
description: Docker Swarm Container Fleet
widget:
type: portainer
url: https://140.44.4.71:9443
env: 3
key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8=
- Infrastructure & Observability:
- NGINX Proxy Manager:
icon: nginx-proxy-manager
href: https://nginx.snarfnet.net
description: Reverse Proxy & SSL Certificates
- Uptime Kuma:
icon: uptime-kuma
href: https://uptime.snarfnet.net
description: Multi-Endpoint Availability Probes
widget:
type: uptimekuma
url: http://140.44.4.28:448
slug: default
- Zabbix:
icon: zabbix.png
href: https://zabbix.snarfnet.net
description: Infrastructure Metrics & Alarms
- Grafana:
icon: grafana.png
href: https://grafana.snarfnet.net
description: Real-Time Metrics & AI Dashboards
widget:
type: grafana
url: https://grafana.snarfnet.net
username: admin
password: Katzir476!!!
- Core Node Glances:
icon: glances.png
href: http://140.44.4.11:61208
description: Host Resource Monitor
- Environment & Weather:
- Tempest Weather Station:
icon: weather.png
href: https://snarfhome.snarfnet.net
description: Backyard Weather Telemetry
widget:
type: homeassistant
url: http://14.4.4.20:8123
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4
custom:
- template: "{{ states('sensor.st_00158555_temperature') | float(0) | round(1) }} °F"
label: Outdoor Temp
- template: "{{ states('sensor.st_00158555_feels_like') | float(0) | round(1) }} °F"
label: Feels Like
- template: "{{ states('sensor.st_00158555_humidity') | float(0) | round(1) }} %"
label: Humidity
- template: "{{ states('sensor.st_00158555_wind_speed') | float(0) | round(1) }} mph"
label: Wind Speed
"""
settings_yaml = """title: SnarfNet Operations & Telemetry
headerStyle: boxed
color: slate
theme: dark
statusStyle: badge
background:
image: https://i.imgur.com/hOzzkCY.jpeg
blur: sm
opacity: 50
brightness: 50
cardBlur: sm
quicklaunch:
searchDescriptions: true
hideInternetSearch: true
hideVisitURL: true
hideVersion: true
layout:
Critical Services:
icon: mdi-shield-check
style: row
columns: 3
GitOps & Automation:
icon: mdi-git
style: row
columns: 4
Storage & Clusters:
icon: mdi-server-network
style: row
columns: 3
Infrastructure & Observability:
icon: mdi-chart-timeline-variant
style: row
columns: 3
Environment & Weather:
icon: mdi-weather-partly-cloudy
style: row
columns: 1
showStats: true
providers:
longhorn:
url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500
"""
widgets_yaml = """- kubernetes:
cluster:
show: true
cpu: true
memory: true
show_names: true
nodes:
show: false
- longhorn:
total: true
nodes: false
- resources:
label: System
backend: resources
cpu: true
expanded: true
memory: true
disk: /
- search:
provider: duckduckgo
target: _blank
"""
bookmarks_yaml = """- Quick Links:
- GitHub:
- abbr: GH
href: https://github.com/
- UniFi Site Manager:
- abbr: UI
href: https://unifi.ui.com/
- Cloudflare:
- abbr: CF
href: https://dash.cloudflare.com/
"""
cm['data']['services.yaml'] = services_yaml
cm['data']['settings.yaml'] = settings_yaml
cm['data']['widgets.yaml'] = widgets_yaml
cm['data']['bookmarks.yaml'] = bookmarks_yaml
cm['data']['kubernetes.yaml'] = "mode: cluster\n"
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
print("Saved clean homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml successfully!")

33
scripts/apply_grafana_dashboards.py Normal file
View File

@@ -0,0 +1,33 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r", encoding="utf-8") as f:
grafana_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_updated.yaml")
stdin.write(grafana_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("Applying Grafana manifests:")
print(run("kubectl apply -f /tmp/grafana_updated.yaml"))
print("\nRestarting Grafana:")
print(run("kubectl rollout restart deployment/grafana -n default"))
print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
print("\nWaiting for Grafana to finish initialization...")
time.sleep(10)
c.close()

20
scripts/apply_grafana_server_side.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
print("Applying server-side:")
print(run("kubectl apply -f /tmp/grafana_full.yaml --server-side --force-conflicts"))
print("\nRollout restart:")
print(run("kubectl rollout restart deployment/grafana -n default"))
print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
c.close()

31
scripts/apply_homepage.py Normal file
View File

@@ -0,0 +1,31 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
if out:
print(out)
if err:
print("[stderr]", err)
return out
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
content = f.read()
stdin, stdout, stderr = c.exec_command('cat > /tmp/homepage.yaml')
stdin.write(content)
stdin.close()
stdout.read()
run("ls -lh /tmp/homepage.yaml")
run("kubectl apply -f /tmp/homepage.yaml")
run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure")
run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s")
c.close()

45
scripts/apply_prom_grafana.py Normal file
View File

@@ -0,0 +1,45 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r") as f:
prom_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml")
stdin.write(prom_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("Applying Prometheus:")
print(run("kubectl apply -f /tmp/prometheus.yaml"))
with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r") as f:
grafana_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_full.yaml")
stdin.write(grafana_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("\nApplying Grafana configuration & dashboards:")
print(run("kubectl apply -f /tmp/grafana_full.yaml"))
print("\nRestarting Grafana:")
print(run("kubectl rollout restart deployment/grafana -n default"))
print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
print("\nChecking Prometheus pod:")
time.sleep(5)
print(run("kubectl get pods -l app.kubernetes.io/name=prometheus"))
c.close()

36
scripts/apply_prom_update.py Normal file
View File

@@ -0,0 +1,36 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f:
prom_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_updated.yaml")
stdin.write(prom_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("Applying Prometheus:")
print(run("kubectl apply -f /tmp/prometheus_updated.yaml"))
print("\nRestarting Prometheus:")
print(run("kubectl rollout restart deployment/prometheus -n default"))
print(run("kubectl rollout status deployment/prometheus -n default --timeout=60s"))
print("\nWaiting 15 seconds for scrape loop...")
time.sleep(15)
print("\nPrometheus active targets count:")
print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets | grep -o '\"health\":\"up\"' | wc -l"))
c.close()

27
scripts/capture_screenshot.py Normal file
View File

@@ -0,0 +1,27 @@
import subprocess
import os
edge_path = r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe"
out_png = os.path.abspath("docs/snarfnet_homelab_topology.png")
html_url = "file:///" + os.path.abspath("docs/snarfnet_homelab_topology.html").replace("\\", "/")
cmd = [
edge_path,
"--headless",
"--disable-gpu",
"--no-sandbox",
"--hide-scrollbars",
"--window-size=1920,2400",
f"--screenshot={out_png}",
html_url
]
print("Running cmd:", " ".join(cmd))
res = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
print("Return code:", res.returncode)
print("Stdout:", res.stdout)
print("Stderr:", res.stderr)
print("File exists?", os.path.exists(out_png))
if os.path.exists(out_png):
print("File size:", os.path.getsize(out_png))

13
scripts/cat_prom_conf.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl exec deployment/prometheus -- cat /etc/prometheus/prometheus.yml"))
c.close()

24
scripts/check_aero_metrics_sources.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Nodes on Aerodrome:")
print(run("kubectl get nodes -o wide"))
print("\nDaemonSets on Aerodrome:")
print(run("kubectl get ds -A"))
print("\nServices matching metrics:")
print(run("kubectl get svc -A | grep -E 'metric|node|cadvisor|exporter'"))
print("\nKubelet metrics on node 140.44.4.11 (port 10250 or 10255):")
print(run("curl -sk https://localhost:10250/metrics/cadvisor | head -n 10"))
c.close()

16
scripts/check_argo_secret.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
import base64
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Argo initial secret:")
print(run("kubectl get secret argocd-initial-admin-secret -n infrastructure -o jsonpath='{.data.password}'"))
c.close()

73
scripts/check_argocd_apps.py Normal file
View File

@@ -0,0 +1,73 @@
import requests
import json
import urllib3
urllib3.disable_warnings()
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json"
}
try:
resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
data = resp.json()
items = data.get("items", [])
print(f"Total Applications: {len(items)}\n")
out_of_sync = []
synced = []
for app in items:
name = app.get("metadata", {}).get("name")
sync_status = app.get("status", {}).get("sync", {}).get("status")
health_status = app.get("status", {}).get("health", {}).get("status")
dest_server = app.get("spec", {}).get("destination", {}).get("server")
dest_ns = app.get("spec", {}).get("destination", {}).get("namespace")
repo = app.get("spec", {}).get("source", {}).get("repoURL")
path = app.get("spec", {}).get("source", {}).get("path")
# Check operation or sync conditions/messages
operation_state = app.get("status", {}).get("operationState", {})
op_phase = operation_state.get("phase")
op_msg = operation_state.get("message")
conditions = app.get("status", {}).get("conditions", [])
info = {
"name": name,
"sync": sync_status,
"health": health_status,
"dest_ns": dest_ns,
"dest_server": dest_server,
"repo": repo,
"path": path,
"op_phase": op_phase,
"op_msg": op_msg,
"conditions": conditions
}
if sync_status != "Synced" or health_status not in ["Healthy", None]:
out_of_sync.append(info)
else:
synced.append(info)
print(f"=== OUT OF SYNC / DEGRADED APPS ({len(out_of_sync)}) ===")
for a in out_of_sync:
print(f"\nApp: {a['name']}")
print(f" Sync: {a['sync']} | Health: {a['health']}")
print(f" Dest: {a['dest_ns']} on {a['dest_server']}")
print(f" Path: {a['path']}")
if a['op_phase']:
print(f" Op Phase: {a['op_phase']}")
if a['op_msg']:
print(f" Op Msg: {a['op_msg']}")
if a['conditions']:
print(f" Conditions: {a['conditions']}")
print(f"\n=== SYNCED APPS ({len(synced)}) ===")
for a in synced:
print(f" {a['name']}: Sync={a['sync']}, Health={a['health']}, NS={a['dest_ns']}")
except Exception as e:
print("Error querying Argo CD API:", e)

29
scripts/check_argocd_resources.py Normal file
View File

@@ -0,0 +1,29 @@
import requests
import json
import urllib3
urllib3.disable_warnings()
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json"
}
resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
data = resp.json()
items = data.get("items", [])
for app in items:
name = app.get("metadata", {}).get("name")
sync = app.get("status", {}).get("sync", {}).get("status")
health = app.get("status", {}).get("health", {}).get("status")
resources = app.get("status", {}).get("resources", [])
out_resources = [r for r in resources if r.get("status") != "Synced" or r.get("health", {}).get("status") not in ["Healthy", None]]
print(f"[{name}] Sync: {sync}, Health: {health}, Total Resources: {len(resources)}")
if out_resources:
print(f" -> Out-of-sync / unhealthy resources:")
for r in out_resources:
print(f" {r.get('kind')}/{r.get('name')}: Sync={r.get('status')}, Health={r.get('health', {}).get('status')}")

24
scripts/check_authentik_proxy_providers.py Normal file
View File

@@ -0,0 +1,24 @@
import urllib.request, ssl, json
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
AK_BASE = "https://authentik.snarfnet.net"
AK_TOKEN = "SnarfNetPermanentAutomationToken2026"
def req(endpoint):
url = f"{AK_BASE}/api/v3/{endpoint}"
r = urllib.request.Request(url, headers={"Authorization": f"Bearer {AK_TOKEN}"})
with urllib.request.urlopen(r, timeout=10, context=ctx) as res:
return json.loads(res.read().decode())
print("=== Authentik Providers (Proxy) ===")
providers = req("providers/proxy/")
for p in providers.get("results", []):
print(p["pk"], p["name"], p.get("internal_host"), p.get("external_host"), p.get("mode"))
print("\n=== Authentik Outposts ===")
outposts = req("outposts/instances/")
for o in outposts.get("results", []):
print(o["pk"], o["name"], o["type"], [p for p in o.get("providers", [])])

16
scripts/check_awx_monitor.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, url, port, accepted_statuscodes_json FROM monitor WHERE name LIKE '%AWX%';" """))
print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM heartbeat WHERE monitor_id IN (SELECT id FROM monitor WHERE name LIKE '%AWX%') ORDER BY id DESC LIMIT 5;" """))
c.close()

17
scripts/check_cm_json.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, o, _ = c.exec_command('kubectl get cm homepage-2-1744068158 -n infrastructure -o json')
import json
cm = json.loads(o.read().decode())
print("Data keys in ConfigMap:", list(cm.get('data', {}).keys()))
print("\n--- settings.yaml ---")
print(cm.get('data', {}).get('settings.yaml'))
print("\n--- widgets.yaml ---")
print(cm.get('data', {}).get('widgets.yaml'))
c.close()

21
scripts/check_dcgm_status.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
out = stdout.read().decode().strip()
return out
time.sleep(5)
print("Pods status:")
print(run("kubectl get pods -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o wide"))
print("Logs from one pod:")
pod = run("kubectl get pod -n ai-stack -l app.kubernetes.io/name=dcgm-exporter -o jsonpath='{.items[0].metadata.name}'")
print(run(f"kubectl logs -n ai-stack {pod} --tail=20"))
c.close()

26
scripts/check_deleted_at.py Normal file
View File

@@ -0,0 +1,26 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
if res:
print(res)
return res
run("""python3 -c "
import sqlite3
con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite')
cur = con.cursor()
for r in cur.execute('SELECT id, user_id, title, pinned, \"order\", deleted_at FROM items LIMIT 10;').fetchall():
print(r)
" """)
c.close()

14
scripts/check_deployment_spec.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl get deployment grafana -o jsonpath='{.spec.template.spec.containers[0].volumeMounts}'"))
c.close()

24
scripts/check_direct_apis.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def test(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
err = e.read().decode().strip()
if res:
print(res[:300])
if err:
print("[stderr]", err[:200])
test("curl -m 3 -s -I http://140.44.4.28:448/ || echo 'Timeout 140.44.4.28:448'")
test("curl -m 3 -s -I http://140.44.4.31:448/ || echo 'Timeout 140.44.4.31:448'")
test("curl -m 3 -s http://140.44.4.70:8050/api/v1/devices || echo 'Failed PeaNUT'")
test("curl -m 3 -s -H 'Authorization: token 47c10c54574131c007302b246352f653facb5062' http://140.44.4.62:90/api/v1/user || echo 'Failed Gitea'")
c.close()

41
scripts/check_down_monitors.py Normal file
View File

@@ -0,0 +1,41 @@
import requests
import json
import urllib3
urllib3.disable_warnings()
try:
r = requests.get("http://140.44.4.28:448/api/status-page/heartbeat/default", verify=False, timeout=10)
data = r.json()
heartbeats = data.get("heartbeatList", {})
down = []
up = []
# Also fetch status page monitor list to map IDs to names
r_sp = requests.get("http://140.44.4.28:448/api/status-page/default", verify=False, timeout=10)
sp_data = r_sp.json()
id_to_name = {}
for group in sp_data.get("publicGroupList", []):
for m in group.get("monitorList", []):
id_to_name[str(m.get("id"))] = m.get("name")
for mid, beats in heartbeats.items():
name = id_to_name.get(str(mid), f"Monitor #{mid}")
if beats:
last = beats[-1]
if last.get("status") == 1:
up.append((name, last.get("ping")))
else:
down.append((name, last.get("msg")))
print(f"Total Monitors: {len(heartbeats)}")
print(f"Up: {len(up)} | Down: {len(down)}")
if down:
print("\nDown Monitors:")
for name, msg in down:
print(f" ❌ {name}: {msg}")
else:
print("\nAll monitors are UP! 🎉")
except Exception as e:
print("Error:", e)

17
scripts/check_gaming_pc.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Ping 140.44.4.80:")
print(run("ping -c 2 -W 1 140.44.4.80 || true"))
print("ARP table for bc:24:11:67:16:cd:")
print(run("ip neigh | grep -i 'bc:24:11:67:16:cd' || arp -a | grep -i 'bc:24:11:67:16:cd' || true"))
c.close()

19
scripts/check_glances_host.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
print("ps aux | grep glances:")
print(run("ps aux | grep glances"))
print("\nnetstat / ss listening:")
print(run("ss -tulpn | grep 61208 || ss -tulpn | grep glances"))
c.close()

13
scripts/check_gotify_files.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("ls -la /mnt/swarm_shared/legacy_volumes/gotify/_data/"))
c.close()

22
scripts/check_gotify_idx.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """import sqlite3
conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/gotify.db')
cur = conn.cursor()
indexes = cur.execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name='applications'").fetchall()
for idx in indexes:
print(idx)
"""
stdin, stdout, stderr = c.exec_command("cat > /tmp/check_idx.py && python3 /tmp/check_idx.py")
stdin.write(py_script)
stdin.close()
print(stdout.read().decode())
print(stderr.read().decode())
c.close()

16
scripts/check_gotify_mount.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
print("Gotify inspect mounts:")
print(run("docker service inspect gotify_gotify --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'"))
c.close()

19
scripts/check_gotify_service.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
print("Gotify service ps:")
print(run("docker service ps gotify_gotify --no-trunc | head -n 10"))
print("\nGotify service logs:")
print(run("docker service logs gotify_gotify --tail=25"))
c.close()

27
scripts/check_grafana_api.py Normal file
View File

@@ -0,0 +1,27 @@
import requests
import json
import urllib3
urllib3.disable_warnings()
auth = ('admin', 'd0KJLZJXUrbIhW1sjU8lrdFYj7VXts26V7Jw7xQr')
try:
r = requests.get('https://grafana.snarfnet.net/api/datasources', auth=auth, verify=False, timeout=10)
print("Datasources Status:", r.status_code)
if r.status_code == 200:
for ds in r.json():
print(f" - ID: {ds.get('id')}, Name: {ds.get('name')}, Type: {ds.get('type')}, URL: {ds.get('url')}, isDefault: {ds.get('isDefault')}")
except Exception as e:
print("Error getting datasources:", e)
try:
r = requests.get('https://grafana.snarfnet.net/api/search', auth=auth, verify=False, timeout=10)
print("Dashboards Status:", r.status_code)
if r.status_code == 200:
dashboards = r.json()
print(f"Total dashboards: {len(dashboards)}")
for d in dashboards[:15]:
print(f" - {d.get('title')} (uri: {d.get('uri')}, type: {d.get('type')})")
except Exception as e:
print("Error getting dashboards:", e)

15
scripts/check_grafana_dashboards_live.py Normal file
View File

@@ -0,0 +1,15 @@
import requests
import json
import urllib3
urllib3.disable_warnings()
auth = ("admin", "Katzir476!!!")
url = "https://grafana.snarfnet.net/api/search"
resp = requests.get(url, auth=auth, verify=False, timeout=10)
print("Status:", resp.status_code)
dashboards = resp.json()
print(f"Total Dashboards Found: {len(dashboards)}")
for d in dashboards:
print(f" - [{d.get('type')}] Title: {d.get('title')} | UID: {d.get('uid')} | URL: {d.get('url')}")

14
scripts/check_grafana_diff.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl diff -f /tmp/grafana_full.yaml || true"))
c.close()

21
scripts/check_grafana_info.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Namespaces on Aerodrome:")
print(run("kubectl get ns"))
print("Grafana pod:")
pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
print(pod)
print("Grafana admin secret:")
print(run("kubectl get secret grafana -o jsonpath='{.data.admin-password}' | base64 -d"))
c.close()

23
scripts/check_grafana_internal.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
print("Pod:", pod)
print("ConfigMap grafana:")
print(run("kubectl get cm grafana -o yaml"))
print("Testing local curl inside pod to localhost:3000/api/health:")
print(run(f"kubectl exec {pod} -- wget -qO- http://localhost:3000/api/health"))
print("Grafana env:")
print(run(f"kubectl exec {pod} -- env | grep -i grafana"))
c.close()

17
scripts/check_grafana_load.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
print("Grafana pod:", pod)
print("Grafana provisioning logs:")
print(run(f"kubectl logs {pod} | grep -i -E 'provision|datasource|dashboard' | tail -n 25"))
c.close()

19
scripts/check_grafana_mounts.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
print(run(f"kubectl exec {pod} -- ls -la /etc/grafana/provisioning/datasources"))
print(run(f"kubectl exec {pod} -- cat /etc/grafana/provisioning/datasources/datasources.yaml"))
print(run(f"kubectl exec {pod} -- ls -la /etc/grafana/provisioning/dashboards"))
print(run(f"kubectl exec {pod} -- cat /etc/grafana/provisioning/dashboards/dashboards.yaml"))
print(run(f"kubectl exec {pod} -- ls -la /var/lib/grafana-dashboards"))
c.close()

13
scripts/check_grafana_rollout.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl get pods -l app.kubernetes.io/name=grafana"))
c.close()

15
scripts/check_grafana_secret.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import base64
import json
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, o, _ = c.exec_command('kubectl get secret grafana -n default -o json')
data = json.loads(o.read().decode()).get('data', {})
for k, v in data.items():
print(k, ":", base64.b64decode(v).decode('utf-8', errors='ignore'))
c.close()

30
scripts/check_grafana_sqlite.py Normal file
View File

@@ -0,0 +1,30 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """import sqlite3
conn = sqlite3.connect('/tmp/grafana_real.db')
cur = conn.cursor()
tables = [t[0] for t in cur.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall()]
for t in tables:
if 'source' in t or 'dash' in t or 'user' in t:
print('Matching table:', t)
print('--- data_source content ---')
for row in cur.execute('SELECT * FROM data_source').fetchall():
print(row)
print('--- dashboard content ---')
for row in cur.execute('SELECT id, title, slug, uid FROM dashboard').fetchall():
print(row)
"""
stdin, stdout, stderr = c.exec_command("cat > /tmp/inspect_grafana.py && python3 /tmp/inspect_grafana.py")
stdin.write(py_script)
stdin.close()
print(stdout.read().decode())
print(stderr.read().decode())
c.close()

17
scripts/check_grafana_success.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
print("Grafana pod:", pod)
print("Provisioning logs:")
print(run(f"kubectl logs {pod} | grep -i -E 'provision|datasource|dashboard'"))
c.close()

31
scripts/check_ha_proxy_js.py Normal file
View File

@@ -0,0 +1,31 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
py_cmd = """
node -e '
const fs = require("fs");
const content = fs.readFileSync("/app/.next/server/pages/api/services/proxy.js", "utf8");
const idx = content.indexOf("states/");
if (idx !== -1) {
console.log("Found states/ at", idx);
console.log(content.slice(Math.max(0, idx - 500), idx + 1000));
} else {
const idx2 = content.indexOf("homeassistant");
console.log("Found homeassistant at", idx2);
console.log(content.slice(Math.max(0, idx2 - 500), idx2 + 1000));
}
'
"""
stdin, stdout, stderr = c.exec_command(f'kubectl exec -n infrastructure {pod} -- sh -c \'{py_cmd}\'')
print(stdout.read().decode())
print(stderr.read().decode())
c.close()

25
scripts/check_harvester_capacity.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko, json
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; kubectl get nodes -o json'"
stdin, stdout, stderr = c.exec_command(cmd)
out = stdout.read().decode()
data = json.loads(out)
total_cpu = 0
total_mem_kb = 0
for n in data['items']:
cpu = int(n['status']['capacity']['cpu'])
mem_str = n['status']['capacity']['memory']
if mem_str.endswith('Ki'):
mem_kb = int(mem_str[:-2])
total_cpu += cpu
total_mem_kb += mem_kb
ram_gib = total_mem_kb / (1024 * 1024)
print(f"Spacedock Harvester: nodes = {len(data['items'])}, cpu = {total_cpu}, ram = {ram_gib:.1f} GiB")
c.close()

21
scripts/check_harvester_prom.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'")
out = stdout.read().decode().strip()
return out
print("=== Check all in cattle-monitoring-system on Harvester ===")
print(run("kubectl get all -n cattle-monitoring-system"))
print("\n=== Check CRDs in monitoring ===")
print(run("kubectl get crd | grep -i monitoring"))
print("\n=== Check Prometheus resources ===")
print(run("kubectl get prometheus -A"))
c.close()

33
scripts/check_harvester_storage.py Normal file
View File

@@ -0,0 +1,33 @@
import paramiko, json
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
full_cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; " + cmd + "'"
stdin, stdout, stderr = c.exec_command(full_cmd)
return stdout.read().decode().strip()
print("=== Blockdevices on Harvester ===")
print(run("kubectl get blockdevices.harvesterhci.io -A -o wide"))
print("\n=== Longhorn storage on Harvester nodes ===")
out = run("kubectl get nodes.longhorn.io -n longhorn-system -o json")
data = json.loads(out)
total_max = 0
total_avail = 0
num_disks = 0
for n in data['items']:
disks = n.get('status', {}).get('diskStatus', {})
for d_name, d_val in disks.items():
num_disks += 1
total_max += d_val.get('storageMaximum', 0)
total_avail += d_val.get('storageAvailable', 0)
tib = total_max / (1024**4)
avail_tib = total_avail / (1024**4)
print(f"Harvester Disks: {num_disks}, Total Storage: {tib:.2f} TiB, Available: {avail_tib:.2f} TiB")
c.close()

20
scripts/check_heartbeats.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print(f"Pod: {pod}")
print("Recent heartbeats summary:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e 'USE kuma; SELECT status, count(*) FROM heartbeat GROUP BY status;'"))
print("\nSample recent heartbeats:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e 'USE kuma; SELECT m.name, h.status, h.msg, h.ping FROM heartbeat h JOIN monitor m ON h.monitor_id = m.id ORDER BY h.time DESC LIMIT 15;'"))
c.close()

31
scripts/check_heimdall_active.py Normal file
View File

@@ -0,0 +1,31 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
if res:
print(res)
return res
run("""python3 -c "
import sqlite3
con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite')
cur = con.cursor()
print('Active items (deleted_at IS NULL):', cur.execute('SELECT count(*) FROM items WHERE deleted_at IS NULL;').fetchone()[0])
print('Deleted items (deleted_at IS NOT NULL):', cur.execute('SELECT count(*) FROM items WHERE deleted_at IS NOT NULL;').fetchone()[0])
print('Active items duplicates:')
print(cur.execute('SELECT count(*), title FROM items WHERE deleted_at IS NULL GROUP BY title HAVING count(*) > 1;').fetchall())
print('Active pinned items:')
for r in cur.execute('SELECT id, title, \"order\", pinned FROM items WHERE deleted_at IS NULL AND pinned = 1 ORDER BY \"order\";').fetchall():
print(r)
" """)
c.close()

22
scripts/check_heimdall_integrity.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
if res:
print(res)
return res
run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'PRAGMA integrity_check;'")
run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1;'")
run("sqlite3 /mnt/nfs_shares/heimdall/www/app.sqlite 'SELECT id, title, pinned, \"order\" FROM items ORDER BY \"order\", title LIMIT 30;'")
c.close()

27
scripts/check_heimdall_pinned.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
if res:
print(res)
return res
run("""python3 -c "
import sqlite3
con = sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite')
cur = con.cursor()
print('Active items breakdown:')
for r in cur.execute('SELECT id, user_id, title, url, pinned, \"order\", deleted_at FROM items WHERE deleted_at IS NULL LIMIT 20;').fetchall():
print(r)
" """)
c.close()

24
scripts/check_heimdall_python.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
err = e.read().decode().strip()
if res:
print(res)
if err:
print("[stderr]", err)
return res
run("which sqlite3 || echo 'sqlite3 not installed on host'")
run("python3 -c \"import sqlite3; con=sqlite3.connect('/mnt/nfs_shares/heimdall/www/app.sqlite'); cur=con.cursor(); print(cur.execute('PRAGMA integrity_check;').fetchall()); print('Total items:', cur.execute('SELECT count(*) FROM items;').fetchone()[0]); print('Duplicates:', cur.execute('SELECT count(*), title FROM items GROUP BY title HAVING count(*) > 1;').fetchall())\"")
c.close()

26
scripts/check_homepage_api.py Normal file
View File

@@ -0,0 +1,26 @@
import requests
import json
import urllib3
urllib3.disable_warnings()
s = requests.Session()
r = s.get('https://homepage.snarfnet.net/api/services', verify=False)
print("Status /api/services:", r.status_code)
try:
data = r.json()
print("Services JSON keys/length:", len(data))
for group in data:
for title, items in group.items():
print(f"Group: {title}")
for item in items:
print(f" - {item.get('name')}: widget={item.get('widget')}")
except Exception as e:
print("Error:", e, r.text[:200])
r2 = s.get('https://homepage.snarfnet.net/api/widgets', verify=False)
print("Status /api/widgets:", r2.status_code)
try:
print("Widgets:", json.dumps(r2.json(), indent=2))
except Exception as e:
print("Error widgets:", e, r2.text[:200])

11
scripts/check_homepage_info.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, o, _ = c.exec_command('kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath="{.spec.template.spec.containers[0].image}"')
print("Image:", o.read().decode())
_, o, _ = c.exec_command('kubectl logs deployment/homepage-2-1744068158 -n infrastructure --tail=20')
print("Logs:\n", o.read().decode())
c.close()

16
scripts/check_homepage_pod_logs.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
print("Logs from Homepage pod:")
print(run(f"kubectl logs -n infrastructure {pod} --tail=30"))
c.close()

21
scripts/check_homepage_rbac.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
print("Pod:", pod)
print("Check can-i list nodes:")
print(run(f"kubectl auth can-i list nodes --as=system:serviceaccount:infrastructure:default"))
print("Check can-i get nodes.metrics.k8s.io:")
print(run(f"kubectl auth can-i get nodes.metrics.k8s.io --as=system:serviceaccount:infrastructure:default"))
c.close()

15
scripts/check_homepage_routes.py Normal file
View File

@@ -0,0 +1,15 @@
import requests
import urllib3
urllib3.disable_warnings()
s = requests.Session()
r = s.get('https://homepage.snarfnet.net', verify=False)
print("Page length:", len(r.text))
import re
apis = re.findall(r'/api/[a-zA-Z0-9_/]+', r.text)
print("Found API routes:", set(apis))
# Also test hitting the Home Assistant custom widget api if any
# In Homepage, the widget endpoint is /api/widgets/homeassistant or similar or /api/services/...
# Let's check pod files to see what routes exist

18
scripts/check_icon_resolver.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'")
# Search for github raw or cdn in server.js or .next
print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'cdn.jsdelivr.net' /app/.next/server/ | head -n 5"))
print(run(f"kubectl exec -n infrastructure {pod} -- grep -rn 'dashboard-icons' /app/src/"))
c.close()

21
scripts/check_kuma_db_backups.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print("Pod:", pod)
print("Check for any .sql or .db files in /app/data:")
print(run(f"kubectl exec -n monitoring {pod} -- find /app/data -name '*.sql*' -o -name '*.db*' -o -name '*.bak*'"))
print("\nCheck Notification in DB:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT id, name, type, is_default FROM notification;'"))
c.close()

23
scripts/check_kuma_details.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print("Monitors count:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT count(id) FROM monitor;'"))
print("\nStatus page count:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT * FROM status_page;'"))
print("\nGroup count:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'SELECT * FROM `group`;'"))
c.close()

27
scripts/check_kuma_live_status.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("=== Uptime Kuma Pods ===")
print(run("kubectl get pods -n monitoring -o wide"))
print("\n=== Uptime Kuma Deployment & Volumes ===")
print(run("kubectl get deployment -n monitoring uptime-kuma-2-1756668961 -o yaml | grep -A 10 volumes"))
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma-2-1756668961 -o jsonpath='{.items[0].metadata.name}'")
print(f"\n=== Current Pod: {pod} ===")
print("\n=== Query MariaDB monitors table ===")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT count(*) FROM monitor; SELECT id, name, type, active FROM monitor;'"))
print("\n=== Files in /app/data ===")
print(run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data"))
c.close()

25
scripts/check_kuma_live_status2.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64"
print(f"=== Querying Pod: {pod} ===")
print("Query MariaDB monitor count:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT count(*) FROM monitor;'"))
print("Query all monitors:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT id, name, type, active FROM monitor;'"))
print("\nFiles in /app/data:")
print(run(f"kubectl exec -n monitoring {pod} -- ls -la /app/data"))
c.close()

20
scripts/check_kuma_logs.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
pod = "uptime-kuma-2-1756668961-7cbccffbbb-mbb64"
print("=== Error log from /app/data/error.log ===")
print(run(f"kubectl exec -n monitoring {pod} -- tail -n 50 /app/data/error.log"))
print("\n=== Recent pod logs ===")
print(run(f"kubectl logs -n monitoring {pod} --tail=60"))
c.close()

17
scripts/check_kuma_mariadb.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-6656d4d5c6-z5cz9 -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT count(*) AS monitor_count FROM monitor; SELECT id, name, type, active FROM monitor LIMIT 15; SELECT * FROM notification;" """
_, stdout, stderr = c.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("[stderr]", err)
c.close()

16
scripts/check_kuma_notif.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb -u root --socket=/app/data/run/mariadb.sock -e 'USE kuma; SELECT * FROM notification;'"))
c.close()

17
scripts/check_kuma_slugs.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print("group:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'DESCRIBE `group`;'"))
print("\nmonitor_group:")
print(run(f"kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root kuma -e 'DESCRIBE monitor_group;'"))
c.close()

20
scripts/check_kuma_sp_schema.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
print("--- STATUS PAGE SCHEMA ---")
print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE status_page; DESCRIBE \`group\`; DESCRIBE monitor_group;" """))
print("\n--- CURRENT GROUPS ---")
print(run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM \`group\`;" """))
c.close()

13
scripts/check_live_cm.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, o, _ = c.exec_command('kubectl get cm homepage-2-1744068158 -n infrastructure -o yaml')
cm_yaml = o.read().decode()
start = cm_yaml.find('settings.yaml:')
print(cm_yaml[start:start+400])
c.close()

17
scripts/check_longhorn_source.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
print('Pod:', pod)
print('Search for longhorn in /app/src or /app:')
print(run(f'kubectl exec -n infrastructure {pod} -- grep -rn "Missing Longhorn URL" /app'))
c.close()

19
scripts/check_longhorn_svc.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
return out
print("Spaceport longhorn-system services:")
print(run("kubectl get svc -n longhorn-system"))
print("\nSpaceport ingress:")
print(run("kubectl get ingress -n longhorn-system"))
c.close()

17
scripts/check_mdi_icons.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'")
print("Checking mdi weather in @mdi/js:")
print(run(f"kubectl exec -n infrastructure {pod} -- node -e \"const mdi = require('@mdi/js'); console.log('mdiWeatherStation:', !!mdi.mdiWeatherStation); console.log('mdiWeatherPartlyCloudy:', !!mdi.mdiWeatherPartlyCloudy);\""))
c.close()

17
scripts/check_monitoring_ns.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
def check_ns(ip, name):
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print(f"=== {name} ({ip}) ===")
_, o, _ = c.exec_command("kubectl get all -n monitoring")
print(o.read().decode())
if name == 'Aerodrome':
_, o, _ = c.exec_command("kubectl get all -n cattle-monitoring-system")
print("cattle-monitoring-system:\n", o.read().decode())
c.close()
check_ns('140.44.4.11', 'Aerodrome')
check_ns('140.44.4.41', 'Spaceport')

13
scripts/check_monitoring_spaceport.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl get all -n monitoring"))
c.close()

18
scripts/check_nas_backups.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Check /mnt/nfs_shares/backups:")
print(run("ls -la /mnt/nfs_shares/backups"))
print("Create /mnt/nfs_shares/game-backups:")
print(run("mkdir -p /mnt/nfs_shares/game-backups && ls -la /mnt/nfs_shares/game-backups"))
c.close()

12
scripts/check_nas_ports.py Normal file
View File

@@ -0,0 +1,12 @@
import socket
for ip in ['14.10.10.71', '14.10.10.72']:
print(f"\nChecking ports on {ip}:")
for port in [22, 80, 111, 161, 443, 445, 2049, 5000, 5001]:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1.0)
res = s.connect_ex((ip, port))
if res == 0:
print(f" Port {port} OPEN")
s.close()

17
scripts/check_nvidia_host.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
return stdout.read().decode().strip()
print("Check nvidia-smi:")
print(run("which nvidia-smi || nvidia-smi"))
print("Check GPU devices:")
print(run("ls -l /dev/nvidia*"))
c.close()

15
scripts/check_nvidia_output.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
return stdout.read().decode().strip()
print("nvidia-smi output:")
print(run("/usr/local/bin/nvidia-smi"))
c.close()

17
scripts/check_pkg_icons.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath='{.items[0].metadata.name}'")
print("Checking package.json dependencies:")
print(run(f"kubectl exec -n infrastructure {pod} -- grep -E 'mdi|icon|simple' /app/package.json"))
c.close()

16
scripts/check_pod_logs.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run('kubectl get pod -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o jsonpath="{.items[0].metadata.name}"')
print('Pod:', pod)
print('Recent pod logs:')
print(run(f'kubectl logs -n infrastructure {pod} --tail=50'))
c.close()

24
scripts/check_ports.py Normal file
View File

@@ -0,0 +1,24 @@
import socket
hosts = [
('140.44.4.11', 9100, 'Aerodrome Master 01'),
('140.44.4.41', 9100, 'Spaceport Master 01'),
('140.44.4.71', 9100, 'Shipyard Swarm'),
('140.44.4.81', 9100, 'Harvester Node 01'),
('140.44.4.81', 9400, 'Harvester Node 01 DCGM'),
('140.44.4.11', 10250, 'Aerodrome Kubelet'),
('140.44.4.41', 10250, 'Spaceport Kubelet'),
('140.44.4.60', 8888, 'Zabbix Web'),
('140.44.4.64', 10051, 'Zabbix Server'),
]
for ip, port, desc in hosts:
s = socket.socket()
s.settimeout(1.5)
try:
s.connect((ip, port))
print(f"[OPEN] {ip}:{port} ({desc})")
s.close()
except Exception as e:
print(f"[CLOSED/ERR] {ip}:{port} ({desc}) - {e}")

14
scripts/check_prom_elsewhere.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
def check_host(ip, name, cmd):
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print(f"=== {name} ({ip}) ===")
_, o, _ = c.exec_command(cmd)
print(o.read().decode())
c.close()
check_host('140.44.4.71', 'Swarm', 'docker service ls; docker ps | grep -i prom')
check_host('140.44.44.121', 'Spacedock', 'kubectl get pods,svc -A | grep -i prom')

29
scripts/check_prom_live_targets.py Normal file
View File

@@ -0,0 +1,29 @@
import paramiko
import json
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
out = run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets")
data = json.loads(out)
print("=== ACTIVE TARGETS ===")
for t in data.get('data', {}).get('activeTargets', []):
print(f"[{t.get('health')}] pool: {t.get('scrapePool')} | job: {t.get('labels', {}).get('job')} | instance: {t.get('labels', {}).get('instance')} | url: {t.get('scrapeUrl')}")
if t.get('lastError'):
print(f" ERROR: {t.get('lastError')}")
print("\n=== SAMPLE NODE EXPORTER QUERY ===")
out = run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(node_cpu_seconds_total{mode=\"idle\"})_by_(instance)'")
print(out)
print("\n=== SAMPLE CADVISOR QUERY ===")
out = run("kubectl exec deployment/prometheus -- wget -qO- 'http://localhost:9090/api/v1/query?query=count(container_cpu_usage_seconds_total)'")
print(out)
c.close()

21
scripts/check_prom_metrics.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("--- PROMETHEUS TARGETS ---")
print(run("curl -s http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | \"\\(.labels.job) -> \\(.scrapeUrl) (\\(.health))\"' | head -n 30"))
print("\n--- SAMPLE NODE METRICS ---")
print(run("curl -s 'http://localhost:9090/api/v1/query?query=node_cpu_seconds_total' | jq -r '.data.result | length'"))
print("\n--- SAMPLE CONTAINER METRICS ---")
print(run("curl -s 'http://localhost:9090/api/v1/query?query=container_cpu_usage_seconds_total' | jq -r '.data.result | length'"))
c.close()

13
scripts/check_prom_pods.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl get pods -A | grep -E 'prom|grafana|node-exporter|cadvisor'"))
c.close()

18
scripts/check_prom_sa.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Prometheus Deployment / ServiceAccount:")
print(run("kubectl get deployment prometheus -o yaml | grep -E 'serviceAccount|image'"))
print("\nClusterRoles / RoleBindings for prometheus:")
print(run("kubectl get clusterrolebindings | grep -i prom"))
c.close()

19
scripts/check_prom_targets.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Prometheus pod status:")
print(run("kubectl get pods -l app.kubernetes.io/name=prometheus"))
print("\nPrometheus targets:")
print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets"))
c.close()

13
scripts/check_prom_targets2.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets"))
c.close()

15
scripts/check_prometheus.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Aerodrome Prometheus pods & svc:")
print(run("kubectl get pods,svc -A | grep -i prometheus"))
c.close()

17
scripts/check_prometheus_spaceport.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Spaceport namespaces:")
print(run("kubectl get ns"))
print("Spaceport Prometheus:")
print(run("kubectl get pods,svc -A | grep -i prometheus"))
c.close()

24
scripts/check_real_gotify.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """import sqlite3
conn = sqlite3.connect('/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db')
cur = conn.cursor()
print('=== REAL GOTIFY DB APPLICATIONS ===')
cols = [c[1] for c in cur.execute('PRAGMA table_info(applications)').fetchall()]
print('Cols:', cols)
for r in cur.execute('SELECT * FROM applications').fetchall():
print(r)
"""
stdin, stdout, stderr = c.exec_command("cat > /tmp/check_real_gotify.py && python3 /tmp/check_real_gotify.py")
stdin.write(py_script)
stdin.close()
print(stdout.read().decode())
print(stderr.read().decode())
c.close()

15
scripts/check_running_mounts.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
pod = run("kubectl get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'")
print(run(f"kubectl get pod {pod} -o jsonpath='{{.spec.containers[0].volumeMounts}}'"))
c.close()

15
scripts/check_sc_aerodrome.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Storage classes on Aerodrome:")
print(run("kubectl get sc"))
c.close()

19
scripts/check_shepherd.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, _ = c.exec_command(cmd)
res = o.read().decode().strip()
if res:
print(res)
run("docker service ps shepherd_shepherd")
run("docker service logs --tail 30 shepherd_shepherd")
c.close()

24
scripts/check_spacedock_prom.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.44.121", username="rancher", password="Katzir476!", timeout=10)
chan = c.invoke_shell()
time.sleep(2)
if chan.recv_ready():
chan.recv(4096)
def run(cmd):
chan.send(cmd + "\r\n")
time.sleep(3)
out = ""
while chan.recv_ready():
out += chan.recv(8192).decode(errors='ignore')
return out
print("Spacedock / Harvester pods with prom:")
print(run("sudo kubectl get pods,svc -A | grep -i prom"))
c.close()

13
scripts/check_status_page.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-8c5dc549c-jfwhc -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, slug, title, published FROM status_page;" """
_, stdout, _ = c.exec_command(cmd)
print(stdout.read().decode())
c.close()

13
scripts/check_status_page_schema.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = """kubectl exec -n monitoring uptime-kuma-2-1756668961-8c5dc549c-jfwhc -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE status_page; DESCRIBE status_page_cname;" """
_, stdout, _ = c.exec_command(cmd)
print(stdout.read().decode())
c.close()

58
scripts/check_storage_and_drives.py Normal file
View File

@@ -0,0 +1,58 @@
import paramiko, json
# 1. Check Spaceport & Aerodrome disks
for name, ip in [('Aerodrome', '140.44.4.11'), ('Spaceport', '140.44.4.41')]:
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, o, _ = c.exec_command('kubectl get nodes.longhorn.io -n longhorn-system -o json')
data = json.loads(o.read().decode())
total_max = 0
total_avail = 0
num_disks = 0
for n in data['items']:
disks = n.get('status', {}).get('diskStatus', {})
for d_name, d_val in disks.items():
num_disks += 1
total_max += d_val.get('storageMaximum', 0)
total_avail += d_val.get('storageAvailable', 0)
tib = total_max / (1024**4)
gib = total_max / (1024**3)
avail_gib = total_avail / (1024**3)
print(f"{name} Longhorn -> Disks: {num_disks}, Total Storage: {tib:.2f} TiB ({gib:.0f} GiB), Available: {avail_gib:.0f} GiB")
c.close()
except Exception as e:
print(f"{name} -> error: {e}")
# 2. Query Zabbix for NAS storage & drives
import urllib.request, ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
def zbx(method, params, auth=None):
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
headers = {"Content-Type": "application/json-rpc"}
if auth:
headers["Authorization"] = f"Bearer {auth}"
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
resp = json.loads(r.read().decode())
return resp.get("result", {})
try:
token = zbx("user.login", {"username": "Admin", "password": "zabbix"})
hosts = zbx("host.get", {"filter": {"name": ["SN-NAS (DS920+)", "SnarfNAS (DS1821+)", "SN-NAS", "SnarfNAS"]}}, token)
print("\nZabbix NAS Hosts:")
for h in hosts:
print(h["hostid"], h["host"], h["name"])
# Get items for storage / disks
items = zbx("item.get", {"hostids": h["hostid"], "search": {"name": "space"}}, token)
for it in items[:5]:
print(" ", it["name"], it.get("lastvalue"))
except Exception as e:
print("Zabbix error:", e)

21
scripts/check_swarm_networks.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
res = o.read().decode().strip()
if res:
print(res)
return res
run("docker network ls")
run("docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.Networks}}'")
c.close()

Some files were not shown because too many files have changed in this diff Show More