Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add cluster auditing, management, and upgrade scripts

Browse Source
This commit is contained in:
Snarf
2026-11-04 17:58:07 -05:00
parent 8117326b14
commit 1b378b29f3
128 changed files with 4744 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/audit_cluster_workloads.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
def inspect_cluster(ip, name):
print(f"\n=======================================================")
print(f"=== {name} WORKLOADS ({ip}) ===")
print(f"=======================================================")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "kubectl get deploy,sts,ds -A -o custom-columns=NAMESPACE:.metadata.namespace,KIND:.kind,NAME:.metadata.name,IMAGES:.spec.template.spec.containers[*].image"
_, out, _ = ssh.exec_command(cmd)
lines = out.read().decode().splitlines()
for l in lines:
if not any(k in l for k in ['kube-system', 'cattle-', 'metallb-system', 'longhorn-system']):
print(l)
ssh.close()
inspect_cluster('140.44.4.11', 'AERODROME')
inspect_cluster('140.44.4.41', 'SPACEPORT')

40
scripts/audit_gitops_manifests.py Normal file
View File

@@ -0,0 +1,40 @@
import os
import re
import paramiko
manifest_root = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests"
print("=== GITOPS MANIFEST IMAGE AUDIT ===")
image_re = re.compile(r'image:\s*([^\s"\']+)', re.IGNORECASE)
chart_re = re.compile(r'chart:\s*([^\s"\']+)', re.IGNORECASE)
version_re = re.compile(r'version:\s*([^\s"\']+)', re.IGNORECASE)
manifest_data = {}
for root, dirs, files in os.walk(manifest_root):
for f in files:
if f.endswith('.yaml') or f.endswith('.yml'):
path = os.path.join(root, f)
rel_path = os.path.relpath(path, manifest_root)
with open(path, 'r', encoding='utf-8', errors='ignore') as fp:
content = fp.read()
images = image_re.findall(content)
charts = chart_re.findall(content)
versions = version_re.findall(content)
if images or charts:
manifest_data[rel_path] = {
'images': list(set(images)),
'charts': list(set(charts)),
'versions': list(set(versions))
}
for rel_path, data in sorted(manifest_data.items()):
print(f"\nFile: {rel_path}")
if data['images']:
print(" Images:", ", ".join(data['images']))
if data['charts']:
print(" Charts:", ", ".join(data['charts']))
if data['versions']:
print(" Versions:", ", ".join(data['versions']))

19
scripts/check_51.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("ss -tulpn | grep ':80 '")
run("ip a")
run("kubectl get pod -A -o wide | grep sninfr4lnxwrk01 | grep pihole || true")
ssh.close()

18
scripts/check_51_nat.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("iptables -t nat -S | grep -E '80\b|31905|31937'")
run("crictl ps")
ssh.close()

15
scripts/check_aerodrome_pods.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
s = paramiko.SSHClient()
s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, _ = s.exec_command(cmd)
print(out.read().decode())
run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
s.close()

19
scripts/check_aerodrome_preflight.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("kubectl get nodes")
run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness | grep -v healthy || true")
ssh.close()

8
scripts/check_aerodrome_quick.py Normal file
View File

@@ -0,0 +1,8 @@
import paramiko
ssh=paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_,o,_=ssh.exec_command('kubectl get nodes')
print(o.read().decode())
ssh.close()

24
scripts/check_all_coder_snapshots.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
nodes = [
("wrk01", "140.44.4.51"),
("wrk02", "140.44.4.52"),
("wrk03", "140.44.4.53"),
("wrk04", "140.44.4.54"),
("wrk05", "140.44.4.55"),
("wrk06", "140.44.4.56"),
]
for name, ip in nodes:
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=4)
_, out, _ = ssh.exec_command("find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path '*/home/coder/.config/coderv2' 2>/dev/null")
res = out.read().decode().strip()
if res:
print(f"[{name} - {ip}] Found:\n{res}")
ssh.close()
except Exception as e:
print(f"[{name}] err: {e}")

19
scripts/check_all_endpoints.py Normal file
View File

@@ -0,0 +1,19 @@
import subprocess
endpoints = [
("Argo CD", "https://argo.snarfnet.net"),
("Rancher Manager", "https://aerodrome.snarfnet.net"),
("AWX Console", "https://awx.snarfnet.net"),
("Authentik SSO", "https://authentik.snarfnet.net"),
("Uptime Kuma", "https://uptime.snarfnet.net/dashboard"),
("Gitea GitOps", "http://140.44.4.51:90"),
("Zabbix Server", "http://140.44.4.51:8888"),
("Portainer Server", "https://140.44.4.71:9443")
]
print("=== ENDPOINT HEALTH CHECK ===")
for name, url in endpoints:
res = subprocess.run(f"curl.exe -k -s -o nul -w '%{{http_code}}' {url}", capture_output=True, text=True, shell=True)
code = res.stdout.strip()
print(f"[{code}] {name} ({url})")

14
scripts/check_argo_apps.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command("kubectl get applications -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
ssh.close()

22
scripts/check_argo_ing.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("kubectl get ingress argo-ing -n infrastructure -o yaml")
run("kubectl get svc pihole-web -n infrastructure -o yaml")
run("kubectl get svc -n infrastructure")
run("kubectl get svc -A | grep -E 'LoadBalancer|140.44.4.62'")
run("curl -k -I -H 'Host: argo.snarfnet.net' http://140.44.4.62")
run("curl -k -I -H 'Host: argo.snarfnet.net' https://140.44.4.62")
ssh.close()

11
scripts/check_argocd_now.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command("kubectl get applications -n infrastructure -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
print(out.read().decode())
ssh.close()

14
scripts/check_cattle_pods.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
def check_cattle(ip, cluster):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print(f"\n[{cluster}] Checking cattle-system pods...")
_, out, _ = ssh.exec_command("kubectl get pods -n cattle-system -o wide")
print(out.read().decode().strip())
ssh.close()
check_cattle('140.44.4.11', 'AERODROME')
check_cattle('140.44.4.41', 'SPACEPORT')

19
scripts/check_chart_versions.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
def check_catalog_versions(ip, cluster_name):
print(f"\n=======================================================")
print(f"=== {cluster_name} AVAILABLE CHART VERSIONS ===")
print(f"=======================================================")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
# We can inspect the Rancher catalog charts
cmd = "kubectl get charts.catalog.cattle.io -A -o custom-columns=REPO:.metadata.namespace,CHART:.metadata.name | head -n 30"
_, out, _ = ssh.exec_command(cmd)
print(out.read().decode())
ssh.close()
check_catalog_versions('140.44.4.11', 'AERODROME')
check_catalog_versions('140.44.4.41', 'SPACEPORT')

11
scripts/check_coder_logs.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command('kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=30')
print(out.read().decode())
ssh.close()

14
scripts/check_coder_page.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command("curl -s https://coder.snarfnet.net/api/v2/users/first")
print("First user status:", out.read().decode())
_, out2, _ = ssh.exec_command("curl -s https://coder.snarfnet.net/api/v2/experiments")
print("Experiments:", out2.read().decode())
ssh.close()

24
scripts/check_ghcr_tags.py Normal file
View File

@@ -0,0 +1,24 @@
import urllib.request
import json
def get_ghcr_tags(package):
# Public unauthenticated ghcr tags endpoint
url = f"https://ghcr.io/v2/{package}/tags/list"
try:
# Get anonymous token
token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{package}:pull"
with urllib.request.urlopen(token_url, timeout=5) as t_resp:
token = json.loads(t_resp.read().decode())['token']
req = urllib.request.Request(url, headers={'Authorization': f'Bearer {token}'})
with urllib.request.urlopen(req, timeout=5) as resp:
data = json.loads(resp.read().decode())
tags = [t for t in data.get('tags', []) if not t.endswith('-dev') and not 'beta' in t]
return tags[-10:]
except Exception as e:
return [f"Err: {e}"]
print("Homepage tags:", get_ghcr_tags("gethomepage/homepage"))
print("Coder tags:", get_ghcr_tags("coder/coder"))
print("Authentik tags:", get_ghcr_tags("goauthentik/server"))

15
scripts/check_grafana_spec.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, _ = ssh.exec_command(cmd)
print(out.read().decode())
run("kubectl get deployment grafana -n default -o jsonpath='{.spec.template.spec.containers[*].name}'")
run("kubectl get deployment grafana -n default -o jsonpath='{.spec.template.spec.containers[*].image}'")
ssh.close()

14
scripts/check_helm_releases.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
def check_helm(ip, name):
print(f"\n=== {name} HELM RELEASES ({ip}) ===")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command("helm list -A")
print(out.read().decode())
ssh.close()
check_helm('140.44.4.11', 'AERODROME')
check_helm('140.44.4.41', 'SPACEPORT')

60
scripts/check_historical_data.py Normal file
View File

@@ -0,0 +1,60 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
# 1. Inspect Gotify database on Swarm
print("=================== GOTIFY (Swarm 140.44.4.71) ===================")
ssh_swm = paramiko.SSHClient()
ssh_swm.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_swm.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run_swm(cmd):
_, out, err = ssh_swm.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
run_swm("cat c:\\Users\\Snarf\\OneDrive\\Desktop\\code\\SnarfCode\\docker-deployments\\gotify.yml 2>/dev/null || true")
run_swm("sudo docker service inspect gotify_gotify --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")
run_swm("sudo docker run --rm -v gotify_gotify_data:/data alpine ls -la /data 2>/dev/null || find /var/lib/docker/volumes/ -name 'gotify.db' 2>/dev/null")
# Query gotify.db if found
run_swm("""python3 -c '
import sqlite3, glob
db_files = glob.glob("/var/lib/docker/volumes/**/gotify.db", recursive=True)
print("Gotify DB files:", db_files)
for db in db_files:
try:
conn = sqlite3.connect(db)
cur = conn.cursor()
print("Tables in", db)
for row in cur.execute("SELECT name FROM sqlite_master WHERE type=\"table\""):
print(" ", row[0])
print("Applications count:", cur.execute("SELECT count(*) FROM applications").fetchone()[0])
print("Clients count:", cur.execute("SELECT count(*) FROM clients").fetchone()[0])
print("Messages count:", cur.execute("SELECT count(*) FROM messages").fetchone()[0])
except Exception as e:
print("Err reading", db, e)
'""")
ssh_swm.close()
# 2. Inspect Semaphore BoltDB on Spaceport
print("\n=================== SEMAPHORE (Spaceport 140.44.4.41) ===================")
ssh_sp = paramiko.SSHClient()
ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run_sp(cmd):
_, out, err = ssh_sp.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
run_sp("kubectl exec -n infrastructure deployment/semaphore -- ls -lh /var/lib/semaphore/database.boltdb")
# Let's inspect projects and templates using semaphore CLI or strings
run_sp("kubectl exec -n infrastructure deployment/semaphore -- strings /var/lib/semaphore/database.boltdb | grep -E 'project|playbook|inventory|repository' | head -n 30")
ssh_sp.close()

15
scripts/check_homepage_spec.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, _ = ssh.exec_command(cmd)
print(out.read().decode())
run("kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath='{.spec.template.spec.containers[*].name}'")
run("kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath='{.spec.template.spec.containers[*].image}'")
ssh.close()

23
scripts/check_jenkins.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("kubectl get pod -n infrastructure jenkins-0 -o wide")
run("kubectl describe pod -n infrastructure jenkins-0 | tail -n 25")
run("curl -k -s -I https://jenkins.snarfnet.net | head -n 5")
run("curl -s -I http://140.44.4.52:101 | head -n 5")
ssh.close()

28
scripts/check_job134.py Normal file
View File

@@ -0,0 +1,28 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
print(f"Job [134]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
events = job.job_events.filter(event__in=['runner_on_ok', 'runner_on_failed', 'playbook_on_stats']).order_by('-id')[:10]
for e in events:
print(f"[{e.event}] Host: {e.event_data.get('host')} - Task: {e.event_data.get('task')}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
_, out, _ = ssh.exec_command('kubectl get nodes')
print("=== KUBERNETES NODES ===")
print(out.read().decode())
ssh.close()

20
scripts/check_job134_end.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
print(job.result_stdout[-1500:])
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

22
scripts/check_job134_explain.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
print(f"Status: {job.status}")
print(f"Job explanation: {job.job_explanation}")
print(f"Result Traceback: {job.result_traceback}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

25
scripts/check_job134_fail.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
failed_events = job.job_events.filter(failed=True)
for e in failed_events:
res = e.event_data.get('res', {})
msg = res.get('msg') or res.get('stderr') or res.get('stdout') or str(res)
print(f"Host: {e.event_data.get('host')}, Task: {e.event_data.get('task')}")
print(f"Details: {msg[:300]}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

21
scripts/check_job134_stdout.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
lines = [l for l in job.result_stdout.split('\\n') if l.strip()]
print('\\n'.join(lines[-35:]))
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

21
scripts/check_job_status.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=131)
print(f"Job [131]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

12
scripts/check_k3s_channels.py Normal file
View File

@@ -0,0 +1,12 @@
import urllib.request
import json
url = "https://update.k3s.io/v1-release/channels"
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json'})
with urllib.request.urlopen(req, timeout=10) as resp:
data = json.loads(resp.read().decode())
for item in data.get('data', []):
print(f"{item.get('id')}: {item.get('latest')}")
except Exception as e:
print("Error:", e)

32
scripts/check_logs_utf8.py Normal file
View File

@@ -0,0 +1,32 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh_sp = paramiko.SSHClient()
ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
ssh_swm = paramiko.SSHClient()
ssh_swm.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_swm.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(ssh, cmd):
print(f"\n--- {cmd} ---")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
print("=== PEANUT LOGS (Swarm) ===")
run(ssh_swm, "sudo docker service logs peanut_peanut --tail=50")
print("\n=== CODER LOGS (Spaceport) ===")
run(ssh_sp, "kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=50")
print("\n=== SEMAPHORE LOGS (Spaceport) ===")
run(ssh_sp, "kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=50")
ssh_sp.close()
ssh_swm.close()

15
scripts/check_longhorn_version.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
def check_lh(ip, cluster):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print(f"\n[{cluster}] Checking Longhorn images...")
_, out, _ = ssh.exec_command("kubectl get daemonset -n longhorn-system longhorn-manager -o jsonpath='{.spec.template.spec.containers[0].image}{\"\\n\"}'")
print("Manager image:", out.read().decode().strip())
_, out, _ = ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state | head -n 10")
print(out.read().decode().strip())
ssh.close()
check_lh('140.44.4.11', 'AERODROME')
check_lh('140.44.4.41', 'SPACEPORT')

11
scripts/check_npm_all_hosts.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -E "server_name|set \\$server|set \\$port" /data/nginx/proxy_host/*.conf""")
print(stdout.read().decode())
ssh.close()

21
scripts/check_npm_argo.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("sudo docker ps | grep nginx")
run("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') find /data -maxdepth 3")
run("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') cat /etc/hosts")
run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -rn "argo" /etc/nginx/ /data/nginx/""")
run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -rn "pihole" /etc/nginx/ /data/nginx/""")
ssh.close()

15
scripts/check_npm_db.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c \"python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print('\\n'.join(str(r) for r in cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE is_deleted=0')))\\\"\""
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
ssh.close()

15
scripts/check_npm_mounts.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
run("sudo docker service inspect nginx-proxy-manager_app --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")
run("sudo docker inspect $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{json .Mounts}}'")
ssh.close()

14
scripts/check_npm_nodes.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
run("sudo docker service ps nginx-proxy-manager_app --filter 'desired-state=running'")
ssh.close()

19
scripts/check_npm_user.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
# Check user table in NPM db
run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print(list(cur.execute('SELECT id, name, email FROM user')))\\\"\"""")
ssh.close()

26
scripts/check_old_coder_layer.py Normal file
View File

@@ -0,0 +1,26 @@
import paramiko
nodes = [
("ctl01", "140.44.4.41"),
("wrk01", "140.44.4.51"),
("wrk02", "140.44.4.52"),
("wrk03", "140.44.4.53"),
("wrk04", "140.44.4.54"),
("wrk05", "140.44.4.55"),
("wrk06", "140.44.4.56"),
]
for name, ip in nodes:
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
_, out, _ = ssh.exec_command("find /run/k3s/containerd/ /var/lib/rancher/k3s/agent/containerd/ -type d -name 'coderv2' 2>/dev/null")
res = out.read().decode().strip()
if res:
print(f"[{name} - {ip}] FOUND coderv2:\n{res}")
else:
print(f"[{name} - {ip}] none")
ssh.close()
except Exception as e:
print(f"[{name} - {ip}] err: {e}")

22
scripts/check_other_apps.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
def query_cluster(ip, name):
print(f"\n=================== CLUSTER {name} ({ip}) ===================")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
run("kubectl get ns")
run("kubectl get deploy,sts -A | grep -E 'awx|authentik|zabbix|coder'")
ssh.close()
query_cluster('140.44.4.11', 'AERODROME')
query_cluster('140.44.4.41', 'SPACEPORT')

22
scripts/check_peanut_now.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
run("sudo docker service logs peanut_peanut --tail=25")
run("curl -s -I http://140.44.4.70:8050/")
run("curl -s http://140.44.4.70:8050/ | grep -i setup || echo 'NO SETUP REDIRECT'")
ssh.close()

19
scripts/check_peanut_storage.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n--- {cmd} ---")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
run("ls -la /mnt/swarm_shared/legacy_volumes/peanut/")
run("ssh snshpd4lnxswm05 'ls -la /mnt/swarm_shared/legacy_volumes/peanut/'")
run("sudo docker service inspect peanut_peanut")
ssh.close()

11
scripts/check_pvc_progress.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command('kubectl get pods,pvc -n infrastructure | grep coder')
print(out.read().decode())
ssh.close()

19
scripts/check_rancher_compat.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("kubectl get settings.management.cattle.io server-version -o yaml")
run("kubectl get pod -n cattle-system -l app=rancher -o jsonpath='{.items[0].spec.containers[*].image}'")
run("kubectl get clusters.management.cattle.io -o custom-columns=NAME:.metadata.name,DISPLAY:.spec.displayName,VERSION:.status.version.gitVersion")
ssh.close()

17
scripts/check_removal_script.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('cat /proc/3712358/cmdline')
print("CMDLINE:", stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('cat /usr/sbin/linux-check-removal || cat /usr/bin/linux-check-removal')
print("SCRIPT:\n", stdout.read().decode())
ssh.close()

19
scripts/check_semaphore_crash.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
errors = err.read().decode().strip()
if res:
print(res)
if errors:
print("ERR:", errors)
run("kubectl logs deployment/semaphore -n infrastructure --tail=40")
ssh.close()

14
scripts/check_semaphore_help.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
ssh.exec_command('kubectl run semaphore-test --image=semaphoreui/semaphore:v2.19.17 --restart=Never -n infrastructure --command -- strings /usr/bin/semaphore')
time.sleep(6)
_, out, _ = ssh.exec_command('kubectl logs semaphore-test -n infrastructure | grep -E "mysql|postgres|bolt" | head -n 30')
print(out.read().decode())
ssh.exec_command('kubectl delete pod semaphore-test -n infrastructure --grace-period=0 --force')
ssh.close()

9
scripts/check_semaphore_stable.py Normal file
View File

@@ -0,0 +1,9 @@
import urllib.request
import json
url = "https://hub.docker.com/v2/repositories/semaphoreui/semaphore/tags?page_size=100"
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req, timeout=5) as resp:
data = json.loads(resp.read().decode())
ga_tags = [t['name'] for t in data.get('results', []) if not any(x in t['name'] for x in ['-rc', '-beta', '-alpha', '-debug', 'latest'])]
print("Semaphore GA tags:", ga_tags[:10])

16
scripts/check_spaceport_health.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
run("kubectl get applications -A -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
ssh.close()

18
scripts/check_spaceport_lb.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("kubectl get ds -A")
run("kubectl get svc traefik -n kube-system -o yaml")
ssh.close()

31
scripts/check_spaceport_packages.py Normal file
View File

@@ -0,0 +1,31 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
spaceport_nodes = [
('sninfr0lnxctl01', '140.44.4.41'),
('sninfr0lnxctl02', '140.44.4.42'),
('sninfr0lnxctl03', '140.44.4.43'),
('sninfr0lnxctl04', '140.44.4.44'),
('sninfr0lnxwrk01', '140.44.4.51'),
('sninfr0lnxwrk02', '140.44.4.52'),
('sninfr0lnxwrk03', '140.44.4.53'),
('sninfr0lnxwrk04', '140.44.4.54'),
('sninfr0lnxwrk05', '140.44.4.55'),
('sninfr0lnxwrk06', '140.44.4.56')
]
for name, ip in spaceport_nodes:
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=5)
stdin, stdout, stderr = ssh.exec_command('apt list --upgradable 2>/dev/null | grep -v "Listing" | wc -l')
cnt = stdout.read().decode().strip()
print(f"[{name}] ({ip}): {cnt} packages upgradeable")
ssh.close()
except Exception as e:
print(f"[{name}] ({ip}): Error connecting: {e}")

9
scripts/check_spaceport_quick.py Normal file
View File

@@ -0,0 +1,9 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command('kubectl get nodes')
print(out.read().decode())
ssh.close()

18
scripts/check_upstream_tags.py Normal file
View File

@@ -0,0 +1,18 @@
import urllib.request
import json
def get_dockerhub_latest(repo, prefix=""):
url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=15"
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req, timeout=5) as resp:
data = json.loads(resp.read().decode())
tags = [t['name'] for t in data.get('results', []) if not t['name'].endswith('-debug') and not 'beta' in t['name']]
return tags[:8]
except Exception as e:
return [f"Err: {e}"]
print("Grafana tags:", get_dockerhub_latest("grafana/grafana"))
print("Uptime Kuma tags:", get_dockerhub_latest("louislam/uptime-kuma"))
print("Semaphore tags:", get_dockerhub_latest("semaphoreui/semaphore"))

16
scripts/check_upstream_tags2.py Normal file
View File

@@ -0,0 +1,16 @@
import urllib.request
import json
def get_tags(repo):
url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=15"
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req, timeout=5) as resp:
data = json.loads(resp.read().decode())
tags = [t['name'] for t in data.get('results', []) if not t['name'].endswith('-debug') and not 'beta' in t['name']]
return tags[:8]
except Exception as e:
return [f"Err: {e}"]
print("Jenkins tags:", get_tags("jenkins/jenkins"))

27
scripts/check_wrk03_fail.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
events = job.job_events.filter(event='runner_on_failed')
for e in events:
print(f"FAILED Host: {e.event_data.get('host')}, Task: {e.event_data.get('task')}")
print(f"Res: {e.event_data.get('res')}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
_, out, _ = ssh.exec_command('kubectl get pods -A --field-selector spec.nodeName=sninfr2lnxwrk03')
print("Pods on wrk03:\n", out.read().decode())
ssh.close()

11
scripts/check_zabbix_status.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command('kubectl get pods -n monitoring -o wide')
print(out.read().decode())
ssh.close()

19
scripts/check_zabbix_tags.py Normal file
View File

@@ -0,0 +1,19 @@
import urllib.request
import json
tags_to_check = [
"zabbix-server-pgsql",
"zabbix-agent2",
"zabbix-web-nginx-pgsql",
"zabbix-web-service"
]
for repo in tags_to_check:
url = f"https://registry.hub.docker.com/v2/repositories/zabbix/{repo}/tags/ubuntu-7.0.32"
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req) as resp:
print(f"[OK] zabbix/{repo}:ubuntu-7.0.32 exists")
except Exception as e:
print(f"[ERR] zabbix/{repo}:ubuntu-7.0.32 -> {e}")

24
scripts/cleanup_unknown_pods.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
for pod in ['pihole-85b7bd696d-ltrvg', 'technitium-dns-6dcf8fc9f6-mxd7p']:
cmd = f'kubectl patch pod -n infra {pod} -p "{{\\"metadata\\":{{\\"finalizers\\":null}}}}" --type=merge'
stdin, stdout, stderr = ssh.exec_command(cmd)
print(f"Patch {pod}:", stdout.read().decode().strip(), stderr.read().decode().strip())
del_cmd = f'kubectl delete pod -n infra {pod} --force --grace-period=0'
stdin, stdout, stderr = ssh.exec_command(del_cmd)
print(f"Delete {pod}:", stdout.read().decode().strip(), stderr.read().decode().strip())
# Check remaining pods in infra
_, out, _ = ssh.exec_command('kubectl get pods -n infra')
print("\nInfra pods:\n", out.read().decode())
ssh.close()

45
scripts/comprehensive_health_sweep.py Normal file
View File

@@ -0,0 +1,45 @@
import urllib.request
import ssl
import sys
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
endpoints = [
("Rancher Console", "https://aerodrome.snarfnet.net"),
("Argo CD Web UI", "https://argo.snarfnet.net"),
("Portainer Server", "https://140.44.4.71:9443"),
("Authentik SSO Portal", "https://authentik.snarfnet.net"),
("AWX Controller", "https://awx.snarfnet.net"),
("Grafana Dashboards", "https://grafana.snarfnet.net"),
("Homepage Dashboard", "https://homepage.snarfnet.net"),
("Uptime Kuma Monitor", "https://uptime.snarfnet.net/dashboard"),
("Jenkins Controller", "https://jenkins.snarfnet.net/login"),
("Coder Workspace", "https://coder.snarfnet.net"),
("Gitea GitOps Service", "http://140.44.4.51:90"),
("Zabbix Web Console", "http://140.44.4.51:8888"),
("Heimdall Portal", "http://140.44.4.71:85"),
("Peanut Dashboard", "http://140.44.4.71:8050"),
("Homebox Inventory", "http://140.44.4.71:3100"),
("Semaphore UI", "https://semaphore.snarfnet.net"),
("NPM Proxy Admin", "http://140.44.4.71:81")
]
print(f"{'SERVICE':<25} {'URL':<40} {'STATUS':<15}")
print("=" * 80)
for name, url in endpoints:
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req, timeout=8, context=ctx) as resp:
code = resp.getcode()
print(f"{name:<25} {url:<40} {code} OK")
except urllib.error.HTTPError as e:
# Some endpoints return 401 or 403 on root if auth required
if e.code in [200, 301, 302, 307, 401, 403]:
print(f"{name:<25} {url:<40} {e.code} Responding")
else:
print(f"{name:<25} {url:<40} {e.code} HTTP Error")
except Exception as e:
print(f"{name:<25} {url:<40} ERR: {e}")

20
scripts/deep_search_coder.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.52', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("Searching for Coder postgres data on wrk02 (140.44.4.52)...")
cmd = """find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -name "coderv2" 2>/dev/null"""
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print("Found coderv2 paths:\n", res)
else:
print("No coderv2 found directly. Searching for pg_control...")
cmd2 = """find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path "*coderv2*" 2>/dev/null | head -n 20"""
_, out2, _ = ssh.exec_command(cmd2)
print("Search result:\n", out2.read().decode().strip())
ssh.close()

68
scripts/diagnose_argo.py Normal file
View File

@@ -0,0 +1,68 @@
import socket
import paramiko
def check_dns():
print("--- DNS Check ---")
try:
ip = socket.gethostbyname("argo.snarfnet.net")
print(f"argo.snarfnet.net resolves to: {ip}")
except Exception as e:
print(f"DNS lookup error: {e}")
def check_npm():
print("\n--- NPM Database Check (140.44.4.71) ---")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
# Check sqlite db or active nginx config in npm
cmd = """sudo python3 -c "import sqlite3; con = sqlite3.connect('/mnt/nfs_shares/nginx_proxy_manager/database.sqlite'); cur = con.cursor(); print(list(cur.execute(\\"SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled, is_deleted FROM proxy_host WHERE domain_names LIKE '%argo%'\\")))" """
stdin, stdout, stderr = ssh.exec_command(cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
print("NPM DB Proxy Host for argo:")
print(out)
if err:
print("Error/stderr:", err)
# Check generated nginx config files
cmd2 = "sudo grep -rn 'argo.snarfnet.net' /mnt/nfs_shares/nginx_proxy_manager/nginx/proxy_host/ || true"
stdin, stdout, stderr = ssh.exec_command(cmd2)
print("NPM config match:")
print(stdout.read().decode().strip())
ssh.close()
def check_spaceport_argo():
print("\n--- Spaceport ArgoCD K8s Check (140.44.4.41) ---")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "kubectl get ingress,svc,pod -n infrastructure | grep -i argo"
stdin, stdout, stderr = ssh.exec_command(cmd)
print("Spaceport Argo Resources:")
print(stdout.read().decode().strip())
# Check Ingress YAML
cmd2 = "kubectl get ingress -n infrastructure -o yaml"
stdin, stdout, stderr = ssh.exec_command(cmd2)
ingress_yaml = stdout.read().decode().strip()
print("\nAll Ingresses in infrastructure:")
for line in ingress_yaml.splitlines():
if "name:" in line or "host:" in line or "service:" in line or "port:" in line:
print(" ", line)
ssh.close()
if __name__ == '__main__':
check_dns()
try:
check_npm()
except Exception as e:
print("NPM check failed:", e)
try:
check_spaceport_argo()
except Exception as e:
print("Spaceport check failed:", e)

52
scripts/diagnose_issues.py Normal file
View File

@@ -0,0 +1,52 @@
import paramiko
def check_spaceport():
print("================ SPACEPORT (Coder & Semaphore) ================")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n--- {cmd} ---")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide")
run("kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=40")
run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o wide")
run("kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=40")
run("curl -k -I https://coder.snarfnet.net")
run("curl -k -I https://semaphore.snarfnet.net")
ssh.close()
def check_swarm():
print("\n================ SWARM (Peanut & NPM) ================")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n--- {cmd} ---")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("sudo docker service ps peanut_peanut --no-trunc")
run("sudo docker service logs peanut_peanut --tail=40")
run("curl -s -I http://140.44.4.70:8050 || curl -s -I http://140.44.4.71:8050")
run("curl -k -I https://peanut.snarfnet.net || true")
run("sudo sqlite3 /mnt/nas_docker/npm/data/database.sqlite 'select id,domain_names,forward_host,forward_port,enabled from proxy_host where domain_names like \"%peanut%\" or domain_names like \"%coder%\" or domain_names like \"%semaphore%\";'")
ssh.close()
check_spaceport()
check_swarm()

23
scripts/diagnose_wrk07.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("=== NODES ===")
_, out, _ = ssh.exec_command('kubectl get nodes')
print(out.read().decode())
print("=== PODS ON SNINFR2LNXWRK07 ===")
_, out, _ = ssh.exec_command('kubectl get pods -A -o wide | grep sninfr2lnxwrk07')
print(out.read().decode())
print("=== ALL PODS NOT RUNNING / NOT COMPLETED ===")
_, out, _ = ssh.exec_command('kubectl get pods -A | grep -v Running | grep -v Completed')
print(out.read().decode())
ssh.close()

93
scripts/export_gitops_manifests.py Normal file
View File

@@ -0,0 +1,93 @@
import paramiko
import yaml
import os
CLEAN_METADATA_KEYS = {'uid', 'resourceVersion', 'generation', 'creationTimestamp', 'managedFields', 'selfLink'}
CLEAN_ANNOTATIONS_PREFIXES = {
'kubectl.kubernetes.io/last-applied-configuration',
'deployment.kubernetes.io/revision',
'field.cattle.io/publicEndpoints'
}
def clean_doc(doc):
if not isinstance(doc, dict):
return doc
doc.pop('status', None)
meta = doc.get('metadata', {})
for k in CLEAN_METADATA_KEYS:
meta.pop(k, None)
ann = meta.get('annotations', {})
keys_to_del = [k for k in ann if any(k.startswith(p) for p in CLEAN_ANNOTATIONS_PREFIXES)]
for k in keys_to_del:
ann.pop(k, None)
if not ann:
meta.pop('annotations', None)
return doc
def dump_resources(ssh, query_cmd, out_file):
print(f"Querying: {query_cmd}")
_, out, err = ssh.exec_command(query_cmd)
raw = out.read().decode('utf-8', errors='replace')
if not raw.strip():
print(f" Warning: No resources found for {query_cmd}")
return
docs = yaml.safe_load_all(raw)
cleaned = []
for d in docs:
if not d: continue
if d.get('kind') == 'List':
for item in d.get('items', []):
cleaned.append(clean_doc(item))
else:
cleaned.append(clean_doc(d))
if cleaned:
os.makedirs(os.path.dirname(out_file), exist_ok=True)
with open(out_file, 'w', encoding='utf-8') as fp:
yaml.dump_all(cleaned, fp, default_flow_style=False, sort_keys=False)
print(f" Saved {len(cleaned)} resources -> {out_file}")
# 1. Connect to Spaceport
print("\n--- Connecting to Spaceport (140.44.4.41) ---")
ssh_sp = paramiko.SSHClient()
ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
base_sp = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport"
# Homepage
dump_resources(ssh_sp, "kubectl get deploy,svc,ing,cm -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o yaml", os.path.join(base_sp, "homepage", "homepage.yaml"))
# Jenkins
dump_resources(ssh_sp, "kubectl get sts,svc,ing -n infrastructure -l app.kubernetes.io/name=jenkins -o yaml", os.path.join(base_sp, "jenkins", "jenkins.yaml"))
# Semaphore
dump_resources(ssh_sp, "kubectl get deploy,svc,ing,cm -n infrastructure -l app.kubernetes.io/name=semaphore -o yaml", os.path.join(base_sp, "semaphore", "semaphore.yaml"))
# Coder
dump_resources(ssh_sp, "kubectl get deploy,svc,ing -n infrastructure -l app.kubernetes.io/name=coder -o yaml", os.path.join(base_sp, "coder", "coder.yaml"))
# Gitea
dump_resources(ssh_sp, "kubectl get deploy,sts,svc,ing,cm -n infrastructure -l app.kubernetes.io/instance=gitea -o yaml", os.path.join(base_sp, "gitea", "gitea.yaml"))
# Zabbix
dump_resources(ssh_sp, "kubectl get deploy,sts,svc,ing -n monitoring -l app.kubernetes.io/instance=zabbix -o yaml", os.path.join(base_sp, "zabbix", "zabbix.yaml"))
ssh_sp.close()
# 2. Connect to Aerodrome
print("\n--- Connecting to Aerodrome (140.44.4.11) ---")
ssh_ae = paramiko.SSHClient()
ssh_ae.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_ae.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
base_ae = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome"
# Grafana
dump_resources(ssh_ae, "kubectl get deploy,svc,ing -n default -l app.kubernetes.io/name=grafana -o yaml", os.path.join(base_ae, "grafana", "grafana.yaml"))
# Uptime Kuma
dump_resources(ssh_ae, "kubectl get deploy,svc,ing -n monitoring -l app.kubernetes.io/name=uptime-kuma -o yaml", os.path.join(base_ae, "uptime-kuma", "uptime-kuma.yaml"))
# AWX
dump_resources(ssh_ae, "kubectl get awx,deploy,svc,ing -n awx -o yaml", os.path.join(base_ae, "awx", "awx.yaml"))
# Authentik
dump_resources(ssh_ae, "kubectl get deploy,sts,svc,ing -n infra -l app.kubernetes.io/instance=authentik -o yaml", os.path.join(base_ae, "authentik", "authentik.yaml"))
ssh_ae.close()
print("\nManifest extraction complete!")

20
scripts/find_coder_data.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.52', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
run("crictl ps -a")
run("find /var/lib/rancher/k3s/agent/containerd/ -type d -name 'coderv2' 2>/dev/null")
run("find /run/k3s/containerd/ -type d -name 'coderv2' 2>/dev/null")
run("find / -name 'database.boltdb' 2>/dev/null")
ssh.close()

36
scripts/fix_and_upgrade_uptime.py Normal file
View File

@@ -0,0 +1,36 @@
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
# 1. Update strategy to Recreate
run("""kubectl patch deployment uptime-kuma-2-1756668961 -n monitoring --type='json' -p='[{"op": "replace", "path": "/spec/strategy", "value": {"type": "Recreate"}}]'""")
# 2. Scale down to 0 to release all locks cleanly
print("[*] Scaling down to 0 to release all volume locks...")
run("kubectl scale deployment uptime-kuma-2-1756668961 -n monitoring --replicas=0")
run("kubectl wait --for=delete pod -l app.kubernetes.io/name=uptime-kuma -n monitoring --timeout=60s || true")
# 3. Set image to 2.5.9
run("kubectl set image deployment/uptime-kuma-2-1756668961 uptime-kuma=louislam/uptime-kuma:2.5.9 -n monitoring")
# 4. Scale up to 1
print("[*] Scaling up to 1...")
run("kubectl scale deployment uptime-kuma-2-1756668961 -n monitoring --replicas=1")
run("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=120s")
run("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma -o wide")
ssh.close()

58
scripts/fix_argo_npm.py Normal file
View File

@@ -0,0 +1,58 @@
import paramiko
def fix_argo():
# 1. Update on swm01 (140.44.4.71)
ssh1 = paramiko.SSHClient()
ssh1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh1.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("--- Backing up SQLite DB and 39.conf ---")
cmd_backup = """
sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "
cp /data/database.sqlite /data/database.sqlite.bak.argo &&
cp /data/nginx/proxy_host/39.conf /data/nginx/proxy_host/39.conf.bak
"
"""
stdin, stdout, stderr = ssh1.exec_command(cmd_backup)
print("Backup stdout:", stdout.read().decode())
print("Backup stderr:", stderr.read().decode())
print("--- Updating SQLite DB ---")
cmd_db = """
sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "
python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); cur.execute('UPDATE proxy_host SET forward_host=\\\'140.44.4.62\\\' WHERE id=39'); con.commit(); con.close(); print('DB updated successfully')\\\"
"
"""
stdin, stdout, stderr = ssh1.exec_command(cmd_db)
print("DB stdout:", stdout.read().decode())
print("DB stderr:", stderr.read().decode())
print("--- Updating 39.conf ---")
cmd_conf = """
sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "
sed -i 's/140.44.4.51/140.44.4.62/g' /data/nginx/proxy_host/39.conf
"
"""
stdin, stdout, stderr = ssh1.exec_command(cmd_conf)
print("Conf sed stdout:", stdout.read().decode())
print("Conf sed stderr:", stderr.read().decode())
print("--- Reloading NGINX on swm01 ---")
stdin, stdout, stderr = ssh1.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
print("Reload swm01 stdout:", stdout.read().decode())
print("Reload swm01 stderr:", stderr.read().decode())
ssh1.close()
# 2. Reload on swm02 (140.44.4.72)
ssh2 = paramiko.SSHClient()
ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh2.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("--- Reloading NGINX on swm02 ---")
stdin, stdout, stderr = ssh2.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
print("Reload swm02 stdout:", stdout.read().decode())
print("Reload swm02 stderr:", stderr.read().decode())
ssh2.close()
if __name__ == '__main__':
fix_argo()

34
scripts/fix_cluster_issues.py Normal file
View File

@@ -0,0 +1,34 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
# 1. Install open-iscsi on sninfr2lnxwrk01 (140.44.4.21)
print("=== INSTALLING OPEN-ISCSI ON WRK01 (140.44.4.21) ===")
ssh_wrk01 = paramiko.SSHClient()
ssh_wrk01.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_wrk01.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh_wrk01.exec_command('apt-get update -qq && apt-get install -y open-iscsi nfs-common && systemctl enable --now iscsid')
print(stdout.read().decode())
print("iscsid status:", end=" ")
_, out, _ = ssh_wrk01.exec_command('systemctl is-active iscsid')
print(out.read().decode().strip())
ssh_wrk01.close()
# 2. Check on Master
ssh_master = paramiko.SSHClient()
ssh_master.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_master.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("=== DELETING CRASHING LONGHORN-MANAGER-SQTL9 POD SO IT RECREATES WITH ISCSID ===")
ssh_master.exec_command('kubectl delete pod -n longhorn-system longhorn-manager-sqtl9')
print("=== CHECKING HELM-INSTALL-TRAEFIK-CRD LOGS ===")
_, out, _ = ssh_master.exec_command('kubectl logs -n kube-system helm-install-traefik-crd-jt9zk --tail=25')
print(out.read().decode())
print("=== CLEANING 130-DAY UNKNOWN PODS IN INFRA ===")
ssh_master.exec_command('kubectl delete pod -n infra pihole-85b7bd696d-ltrvg technitium-dns-6dcf8fc9f6-mxd7p --force --grace-period=0')
ssh_master.close()

41
scripts/fix_traefik_and_verify.py Normal file
View File

@@ -0,0 +1,41 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
print("=== FIXING TRAEFIK CRD LABELS & ANNOTATIONS ===")
annotate_cmd = """
for crd in $(kubectl get crd -o name | grep 'traefik.io'); do
echo "Patching $crd..."
kubectl annotate $crd meta.helm.sh/release-name=traefik-crd meta.helm.sh/release-namespace=kube-system --overwrite
kubectl label $crd app.kubernetes.io/managed-by=Helm --overwrite
done
"""
stdin, stdout, stderr = ssh.exec_command(annotate_cmd)
print(stdout.read().decode())
print("=== DELETING FAILED HELM INSTALL JOBS TO RETRIGGER ===")
ssh.exec_command('kubectl delete job -n kube-system helm-install-traefik-crd helm-install-traefik')
print("Waiting 15 seconds for Helm controller to reconcile...")
time.sleep(15)
print("=== CHECKING LONGHORN-MANAGER ON WRK01 ===")
_, out, _ = ssh.exec_command('kubectl get pods -n longhorn-system -l app=longhorn-manager -o wide')
print(out.read().decode())
print("=== CHECKING ALL PODS IN KUBE-SYSTEM ===")
_, out, _ = ssh.exec_command('kubectl get pods -n kube-system | grep traefik')
print(out.read().decode())
print("=== CHECKING ANY REMAINING CRASHING PODS ===")
_, out, _ = ssh.exec_command('kubectl get pods -A | grep -v Running | grep -v Completed')
print(out.read().decode())
ssh.close()

18
scripts/fix_uptime_kuma.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
print(out.read().decode().strip())
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("kubectl get deployment uptime-kuma-2-1756668961 -n monitoring -o yaml | grep -A 4 strategy:")
run("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma")
ssh.close()

21
scripts/get_coder_env.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import json
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, err = ssh.exec_command('kubectl get pod -n infrastructure -l app.kubernetes.io/name=coder -o json')
pod = json.loads(out.read().decode())
if pod.get('items'):
container = pod['items'][0]['spec']['containers'][0]
print("Image:", container.get('image'))
print("Env:")
for e in container.get('env', []):
print(f" {e.get('name')}: {e.get('value', e.get('valueFrom'))}")
print("VolumeMounts:")
for vm in container.get('volumeMounts', []):
print(f" {vm.get('name')} -> {vm.get('mountPath')}")
ssh.close()

15
scripts/inspect_argocd_apps.py Normal file
View File

@@ -0,0 +1,15 @@
import glob
import yaml
files = glob.glob(r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\argocd\**\*.yaml", recursive=True)
for f in files:
with open(f, 'r') as fp:
docs = yaml.safe_load_all(fp)
for doc in docs:
if not doc: continue
if doc.get('kind') == 'Application':
name = doc.get('metadata', {}).get('name')
dest = doc.get('spec', {}).get('destination', {})
src = doc.get('spec', {}).get('source', {})
print(f"App: {name:<30} Path: {src.get('path'):<35} Server: {dest.get('server'):<35} NS: {dest.get('namespace')}")

18
scripts/inspect_authentik.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
run("kubectl get deploy,sts,pod -n infra -o wide")
run("kubectl get deploy authentik-server -n infra -o yaml | grep -E 'image:|strategy:' -A 5")
ssh.close()

23
scripts/inspect_coder_data_to_persist.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder")
run("kubectl exec -n infrastructure deployment/coder -- ls -la /home/coder")
run("kubectl exec -n infrastructure deployment/coder -- du -sh /home/coder/.config/coderv2")
run("curl -s https://coder.snarfnet.net/api/v2/users/first")
ssh.close()

21
scripts/inspect_coder_helm.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import base64
import gzip
import json
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command('kubectl get secret -n infrastructure sh.helm.release.v1.coder.v5 -o jsonpath="{.data.release}"')
raw = out.read().decode().strip()
if raw:
data = base64.b64decode(raw)
data = base64.b64decode(data) # helm encodes twice
decomp = gzip.decompress(data)
rel = json.loads(decomp)
print("Chart:", rel.get('chart', {}).get('metadata', {}).get('version'))
print("Values:", json.dumps(rel.get('config', {}), indent=2))
ssh.close()

19
scripts/inspect_frontend_fd.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('ls -l /proc/3712358/fd')
print("--- FD OF LINUX-CHECK-REMOVAL ---")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('ls -l /proc/3712345/fd')
print("--- FD OF FRONTEND ---")
print(stdout.read().decode())
ssh.close()

23
scripts/inspect_gotify_db.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """sudo python3 -c '
import sqlite3
conn = sqlite3.connect("/mnt/swarm_shared/legacy_volumes/gotify/gotify.db")
cur = conn.cursor()
print("Applications:", cur.execute("SELECT id, name, description FROM applications").fetchall())
print("Clients:", cur.execute("SELECT id, name FROM clients").fetchall())
print("Messages count:", cur.execute("SELECT count(*) FROM messages").fetchone()[0])
print("Recent messages:", cur.execute("SELECT id, title, message, date FROM messages ORDER BY id DESC LIMIT 5").fetchall())
print("Users:", cur.execute("SELECT id, name, admin FROM users").fetchall())
'"""
_, out, err = ssh.exec_command(py_script)
print(out.read().decode())
print("ERR:", err.read().decode())
ssh.close()

28
scripts/inspect_helm_and_images.py Normal file
View File

@@ -0,0 +1,28 @@
import paramiko
ssh_sp = paramiko.SSHClient()
ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
ssh_ae = paramiko.SSHClient()
ssh_ae.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_ae.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(ssh, name, cmd):
print(f"\n--- {name}: {cmd} ---")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
run(ssh_sp, "SPACEPORT", "helm list -A")
run(ssh_sp, "SPACEPORT", "kubectl get deploy -n monitoring -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'")
run(ssh_ae, "AERODROME", "helm list -A")
run(ssh_ae, "AERODROME", "kubectl get deploy -n infra -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'")
run(ssh_ae, "AERODROME", "kubectl get deploy -n awx -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'")
run(ssh_ae, "AERODROME", "kubectl get awx -n awx -o yaml | grep -E 'image|version'")
ssh_sp.close()
ssh_ae.close()

11
scripts/inspect_homepage_k8s.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command('kubectl get all,ing,cm,pvc -n infrastructure | grep -i homepage')
print(out.read().decode())
ssh.close()

24
scripts/inspect_job_events.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=131)
print(f"Result Traceback: {job.result_traceback}")
events = job.job_events.filter(event__in=['runner_on_ok', 'runner_on_failed', 'playbook_on_stats']).order_by('-id')[:20]
for e in events:
print(f"[{e.event}] Host: {e.event_data.get('host')} - Task: {e.event_data.get('task')}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

23
scripts/inspect_jt10.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
py_code = """
from awx.main.models import JobTemplate
jt = JobTemplate.objects.get(id=10)
print(f"Name: {jt.name}")
print(f"Playbook: {jt.playbook}")
print(f"Inventory: {jt.inventory.name if jt.inventory else None}")
print(f"Credential: {[c.name for c in jt.credentials.all()]}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

18
scripts/inspect_npm_hosts.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """sudo python3 -c '
import sqlite3
conn = sqlite3.connect("/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite")
cur = conn.cursor()
for row in cur.execute("select id,domain_names,forward_host,forward_port,enabled from proxy_host"):
print(row)
'"""
_, out, err = ssh.exec_command(py_script)
print(out.read().decode())
print("ERR:", err.read().decode())
ssh.close()

17
scripts/inspect_peanut_simple.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command("sudo docker service inspect peanut_peanut --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")
print("Mounts:", out.read().decode())
_, out2, _ = ssh.exec_command("ls -la /mnt/swarm_shared/legacy_volumes/peanut/")
print("Files:", out2.read().decode())
_, out3, _ = ssh.exec_command("cat /mnt/swarm_shared/legacy_volumes/peanut/settings.yml")
print("Settings:", out3.read().decode())
ssh.close()

19
scripts/inspect_peanut_stack.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
run("sudo docker service inspect peanut_peanut --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'")
run("find /mnt/ -name 'settings.yml' 2>/dev/null")
run("find /mnt/ -name '*peanut*' 2>/dev/null")
ssh.close()

29
scripts/inspect_portainer.py Normal file
View File

@@ -0,0 +1,29 @@
import paramiko
import sys
import json
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('sudo docker inspect portainer')
data = json.loads(stdout.read().decode())[0]
print("Container Name:", data["Name"])
print("Image:", data["Config"]["Image"])
print("Command:", data["Config"]["Cmd"])
print("Mounts:")
for m in data.get("Mounts", []):
print(" ", m.get("Source"), "->", m.get("Destination"))
stdin, stdout, stderr = ssh.exec_command('sudo docker exec portainer /portainer --version || sudo docker inspect --format="{{.Config.Labels}}" portainer')
print("Version/Labels:", stdout.read().decode().strip())
# Check portainer_agent service
stdin, stdout, stderr = ssh.exec_command('sudo docker service inspect portainer_agent --format="{{.Spec.TaskTemplate.ContainerSpec.Image}}"')
print("Agent Service Image:", stdout.read().decode().strip())
ssh.close()

23
scripts/inspect_prerm.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('pstree -p 3700051 || ps -ef | grep 3712344')
print("--- PROCESS TREE ---")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('cat /proc/3712344/cmdline; echo ""; cat /proc/3712344/wchan')
print("--- WCHAN ---")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('cat /var/lib/dpkg/info/linux-image-6.6.62+rpt-rpi-v8.prerm')
print("--- PRERM SCRIPT ---")
print(stdout.read().decode())
ssh.close()

20
scripts/inspect_rancher.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('kubectl get deployment -n cattle-system rancher -o jsonpath="{.spec.template.spec.containers[0].image}"')
print("Rancher image:", stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('kubectl get clusters.management.cattle.io -o custom-columns=NAME:.metadata.name,DISPLAY:.spec.displayName,K8S_VER:.status.version.gitVersion')
print("\nRancher Managed Clusters:\n", stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('kubectl get nodes')
print("Direct kubectl nodes on Aerodrome:\n", stdout.read().decode())
ssh.close()

23
scripts/inspect_semaphore_boltdb.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_script = """python3 -c '
# Simple BoltDB reader without external dependencies
# Or let us inspect with strings or hexdump
import os
print("BoltDB file size:", os.path.getsize("/var/lib/semaphore/database.boltdb"))
'"""
# In the pod, let us see if boltdb has project records
cmd = """kubectl exec -n infrastructure deployment/semaphore -c semaphore -- /bin/sh -c '
strings /var/lib/semaphore/database.boltdb | grep -E "name|title|playbook|project" | head -n 40
'"""
_, out, err = ssh.exec_command(cmd)
print("Strings from Semaphore BoltDB:")
print(out.read().decode())
ssh.close()

28
scripts/inspect_spaceport_workloads.py Normal file
View File

@@ -0,0 +1,28 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
# Check Coder
run("kubectl get deploy,sts,pod -n infrastructure -l app.kubernetes.io/name=coder -o wide")
run("kubectl get deploy,sts,pod -n infrastructure | grep -E 'coder|gitea|zabbix'")
# Inspect Gitea details
run("kubectl get deploy,sts -n infrastructure -o wide")
# Check Coder images
run("kubectl get deploy -n infrastructure -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'")
ssh.close()

23
scripts/inspect_swarm_digests.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("sudo docker service ps heimdall_heimdall --no-trunc")
run("sudo docker service ps gotify_gotify --no-trunc")
run("sudo docker service ps homebox_homebox --no-trunc")
run("sudo docker service ps peanut_peanut --no-trunc")
ssh.close()

21
scripts/inspect_swarm_services.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
e = err.read().decode().strip()
if e:
print("ERR:", e)
run("sudo docker service ls")
run("sudo docker stack ls")
ssh.close()

26
scripts/inspect_swarm_status.py Normal file
View File

@@ -0,0 +1,26 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('sudo docker node ls')
print("=== DOCKER NODES ===")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('sudo docker service ls')
print("=== DOCKER SERVICES ===")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('ip addr show | grep 140.44.4')
print("=== IP ADDRESSES ON SWM01 ===")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('systemctl is-active keepalived')
print("Keepalived status:", stdout.read().decode().strip())
ssh.close()

23
scripts/inspect_wrk07_process.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
stdin, stdout, stderr = ssh.exec_command('ps aux | grep -v "grep" | grep -i apt')
print("--- APT PROCESSES ---")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('ps aux | grep -v "grep" | grep -i dpkg')
print("--- DPKG PROCESSES ---")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command('uptime')
print("--- UPTIME ---")
print(stdout.read().decode())
ssh.close()

17
scripts/inspect_zabbix_deployments.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode().strip()
if res:
print(res)
run("kubectl get deploy -n monitoring -o jsonpath='{range .items[*]}{\"DEPLOY: \"}{.metadata.name}{\"\\n\"}{range .spec.template.spec.containers[*]}{\" container: \"}{.name}{\" image: \"}{.image}{\"\\n\"}{end}{\"\\n\"}{end}'")
ssh.close()

22
scripts/launch_k3s_upgrade.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_launch = """
from awx.main.models import JobTemplate
jt = JobTemplate.objects.get(id=12)
job = jt.create_unified_job()
job.signal_start()
print(f"Launched Job [{job.id}] for Template '{jt.name}': Status={job.status}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_launch}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

24
scripts/launch_os_patching_job.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_launch = """
from awx.main.models import JobTemplate
jt = JobTemplate.objects.get(id=10)
job = jt.create_unified_job()
job.signal_start()
print(f"Launched Job [{job.id}] for Template '{jt.name}': Status={job.status}")
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_launch}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

20
scripts/list_awx_templates.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
py_code = """
from awx.main.models import JobTemplate
for jt in JobTemplate.objects.all():
print(f'ID {jt.id}: {jt.name} (playbook: {jt.playbook})')
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

131
scripts/persist_coder_storage.py Normal file
View File

@@ -0,0 +1,131 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
e = err.read().decode('utf-8', errors='replace').strip()
if e:
print("ERR:", e)
print("[1] Creating backup directory on Spaceport master...")
run("mkdir -p /var/backups/coder_home")
print("[2] Getting current Coder pod name...")
_, out, _ = ssh.exec_command("kubectl get pod -n infrastructure -l app.kubernetes.io/name=coder -o jsonpath='{.items[0].metadata.name}'")
pod_name = out.read().decode().strip()
print(f"Active Coder pod: {pod_name}")
print("[3] Copying /home/coder from active pod to /var/backups/coder_home...")
run(f"kubectl exec -n infrastructure {pod_name} -- tar -czf /tmp/coder_backup.tar.gz -C /home/coder .")
run(f"kubectl cp infrastructure/{pod_name}:/tmp/coder_backup.tar.gz /var/backups/coder_home/coder_backup.tar.gz")
run("ls -lh /var/backups/coder_home/coder_backup.tar.gz")
print("[4] Creating PersistentVolumeClaim 'coder-data' (10Gi, Longhorn)...")
pvc_yaml = """cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: coder-data
namespace: infrastructure
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 10Gi
EOF"""
run(pvc_yaml)
print("[5] Launching temporary population pod to unpack backup into PVC...")
helper_yaml = """cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: coder-pvc-populator
namespace: infrastructure
spec:
restartPolicy: Never
containers:
- name: populator
image: alpine:latest
command: ["sleep", "300"]
volumeMounts:
- name: data
mountPath: /data
volumes:
- name: data
persistentVolumeClaim:
claimName: coder-data
EOF"""
run(helper_yaml)
print("Waiting for populator pod to be Ready...")
run("kubectl wait --for=condition=Ready pod/coder-pvc-populator -n infrastructure --timeout=60s")
print("[6] Copying tarball into populator pod and unpacking with correct permissions (1000:1000)...")
run("kubectl cp /var/backups/coder_home/coder_backup.tar.gz infrastructure/coder-pvc-populator:/data/coder_backup.tar.gz")
run("kubectl exec -n infrastructure coder-pvc-populator -- tar -xzf /data/coder_backup.tar.gz -C /data/")
run("kubectl exec -n infrastructure coder-pvc-populator -- rm /data/coder_backup.tar.gz")
run("kubectl exec -n infrastructure coder-pvc-populator -- chown -R 1000:1000 /data")
run("kubectl exec -n infrastructure coder-pvc-populator -- ls -la /data")
print("[7] Deleting helper populator pod...")
run("kubectl delete pod coder-pvc-populator -n infrastructure --grace-period=0 --force")
print("[8] Patching Coder deployment with persistent volume and strategy=Recreate...")
patch_json = '''kubectl patch deployment coder -n infrastructure --patch '
{
"spec": {
"strategy": {
"type": "Recreate",
"rollingUpdate": null
},
"template": {
"spec": {
"containers": [
{
"name": "coder",
"volumeMounts": [
{
"name": "coder-data",
"mountPath": "/home/coder"
}
]
}
],
"volumes": [
{
"name": "coder-data",
"persistentVolumeClaim": {
"claimName": "coder-data"
}
}
]
}
}
}
}' '''
run(patch_json)
print("[9] Awaiting Coder rollout...")
run("kubectl rollout status deployment/coder -n infrastructure --timeout=180s")
run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide")
print("[10] Verifying Coder health and database persistence...")
run("curl -s https://coder.snarfnet.net/api/v2/users/first")
run("curl -k -s -I https://coder.snarfnet.net | head -n 8")
ssh.close()

11
scripts/read_npm_39.py Normal file
View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') cat /data/nginx/proxy_host/39.conf")
print(stdout.read().decode())
ssh.close()

30
scripts/refresh_argocd.py Normal file
View File

@@ -0,0 +1,30 @@
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, err = ssh.exec_command(cmd)
res = out.read().decode('utf-8', errors='replace').strip()
if res:
print(res)
e = err.read().decode('utf-8', errors='replace').strip()
if e:
print("ERR:", e)
print("[*] Annotating applications with argocd.argoproj.io/refresh: hard...")
# Trigger hard refresh on root and workload apps
run("kubectl annotate app -n infrastructure root-app-of-apps aerodrome-workloads spaceport-workloads argocd.argoproj.io/refresh=hard --overwrite")
run("kubectl annotate app -n infrastructure --all argocd.argoproj.io/refresh=hard --overwrite")
print("[*] Waiting 15 seconds for ArgoCD repo-server and controller reconciliation...")
time.sleep(15)
print("[*] Checking ArgoCD applications sync & health status...")
run("kubectl get applications -n infrastructure -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status,MESSAGE:.status.conditions[0].message")
ssh.close()

23
scripts/search_job134_fail.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
py_check = """
from awx.main.models import Job
job = Job.objects.get(id=134)
lines = job.result_stdout.split('\\n')
for i, l in enumerate(lines):
if 'failed:' in l.lower() or 'fatal:' in l.lower():
print('\\n'.join(lines[max(0, i-5):min(len(lines), i+15)]))
"""
cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF"
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
ssh.close()

Some files were not shown because too many files have changed in this diff Show More