Snarf/SnarfCode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add infrastructure health check and management automation scripts

Browse Source
This commit is contained in:
Snarf
2026-11-05 10:13:07 -05:00
parent 14fac7151f
commit 1fce3f2488
79 changed files with 1391 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
scripts/apply_coredns_spaceport.py Normal file
View File

@@ -0,0 +1,48 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
cm_yaml = """cat << 'EOF' | k3s kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns-custom
namespace: kube-system
data:
snarfnet.override: |
hosts {
140.44.4.34 aerodrome.snarfnet.net
140.44.4.90 harvester.spacedock.snarfnet.net
fallthrough
}
EOF
"""
print("Applying coredns-custom on Spaceport...")
stdin, stdout, stderr = client.exec_command(cm_yaml)
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
print("\nRestarting coredns deployment on Spaceport...")
client.exec_command("k3s kubectl -n kube-system rollout restart deployment/coredns")
client.exec_command("k3s kubectl -n kube-system rollout status deployment/coredns --timeout=60s")
print("\nRestarting cattle-cluster-agent on Spaceport...")
client.exec_command("k3s kubectl -n cattle-system rollout restart deployment/cattle-cluster-agent")
client.exec_command("k3s kubectl -n cattle-system rollout status deployment/cattle-cluster-agent --timeout=60s")
time.sleep(5)
print("\nChecking cattle-cluster-agent logs:")
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent --tail=25")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_13_conf.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /data/nginx/proxy_host/13.conf | grep server_name')
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_71_containers.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_aero_health.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('k3s kubectl get pods -n cattle-system')
print("=== AERODROME CATTLE-SYSTEM PODS ===")
print(stdout.read().decode('utf-8'))
client.close()

29
scripts/check_after_reboot.py Normal file
View File

@@ -0,0 +1,29 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
print("=== CHECKING VETH ON SNINFR0LNXCTL03 ===")
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("modprobe veth; lsmod | grep veth; systemctl is-active k3s-server || systemctl is-active k3s")
print(stdout.read().decode('utf-8'))
client.close()
time.sleep(5)
print("=== CHECKING PODS IN CATTLE-SYSTEM ON SPACEPORT ===")
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide")
print(stdout.read().decode('utf-8'))
print("=== CHECKING POD EVENTS ===")
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system describe pod -l app=cattle-cluster-agent | grep -A 10 Events:")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_agent_deploy.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get deployment cattle-cluster-agent -o yaml")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_agent_logs.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent --tail=100")
print("=== SPACEPORT CATTLE-CLUSTER-AGENT LOGS ===")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_agent_logs_tail.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=20")
print("=== AGENT LOGS ===")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_agent_now.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=25")
print(stdout.read().decode('utf-8'))
client.close()

27
scripts/check_all_tls.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Check certificates in Aerodrome across all namespaces
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secrets -A --field-selector=type=kubernetes.io/tls
""")
print("=== TLS SECRETS ON AERODROME ===")
print(stdout.read().decode('utf-8'))
client.close()
# Check certificates on Spaceport across all namespaces
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secrets -A --field-selector=type=kubernetes.io/tls
""")
print("=== TLS SECRETS ON SPACEPORT ===")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_blkid.py
View File

@@ -11,3 +11,4 @@ client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10
stdin, stdout, stderr = client.exec_command("blkid")
print(stdout.read().decode('utf-8'))
client.close()

20
scripts/check_cattle_all.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide")
print("=== SPACEPORT CATTLE-SYSTEM PODS ===")
print(stdout.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get deployment cattle-cluster-agent")
print("=== DEPLOYMENT STATUS ===")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_cattle_spaceport_clean.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_certbot.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) certbot certificates")
print("OUT:", stdout.read().decode('utf-8'))
print("ERR:", stderr.read().decode('utf-8'))
client.close()

14
scripts/check_cluster_cond.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('k3s kubectl get clusters.provisioning.cattle.io -A')
print("=== PROVISIONING CLUSTERS ===")
print(stdout.read().decode('utf-8'))
client.close()

16
scripts/check_cluster_ready.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.displayName}{"\tConnected: "}{.status.agentImage}{"\tReady: "}{range .status.conditions[?(@.type=="Ready")]}{.status}{" - "}{.message}{end}{"\n"}{end}'
""")
print(stdout.read().decode('utf-8'))
client.close()

25
scripts/check_cluster_tokens.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
import yaml
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusterregistrationtokens.management.cattle.io -A -o yaml")
data = yaml.safe_load(stdout.read().decode('utf-8'))
client.close()
for item in data.get('items', []):
cluster = item.get('spec', {}).get('clusterName')
token = item.get('status', {}).get('token')
cmd = item.get('status', {}).get('command')
manifest_url = item.get('status', {}).get('manifestUrl')
print(f"\nCluster: {cluster}")
print(f"Token: {token}")
print(f"Command: {cmd}")
print(f"Manifest URL: {manifest_url}")

26
scripts/check_cluster_yaml.py Normal file
View File

@@ -0,0 +1,26 @@
import paramiko
import sys
import yaml
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io -o yaml")
data = yaml.safe_load(stdout.read().decode('utf-8'))
client.close()
for item in data.get('items', []):
name = item.get('metadata', {}).get('name')
disp = item.get('spec', {}).get('displayName')
print(f"\n==========================================")
print(f"Cluster: {disp} ({name})")
print(f"==========================================")
conditions = item.get('status', {}).get('conditions', [])
for c in conditions:
if c.get('status') != 'True' or c.get('type') in ['Ready', 'Connected']:
print(f" [{c.get('type')}]: status={c.get('status')} reason={c.get('reason')} msg={c.get('message')}")

1
scripts/check_cmdline.py
View File

@@ -17,3 +17,4 @@ print("DISKS:")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_coredns.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n kube-system get configmap coredns -o yaml")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_coredns_logs.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n kube-system logs -l k8s-app=kube-dns --tail=30")
print(stdout.read().decode('utf-8'))
client.close()

18
scripts/check_ctl02_uptime.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect("140.44.4.42", username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = client.exec_command("uptime; uname -r; modprobe veth; lsmod | grep veth")
print("OUTPUT from ctl02:")
print(stdout.read().decode('utf-8'))
client.close()
except Exception as e:
print("Connect error:", e)

25
scripts/check_current_status.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
import yaml
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io -o yaml")
data = yaml.safe_load(stdout.read().decode('utf-8'))
client.close()
for item in data.get('items', []):
name = item.get('metadata', {}).get('name')
disp = item.get('spec', {}).get('displayName')
print(f"\n==========================================")
print(f"Cluster: {disp} ({name})")
print(f"==========================================")
for c in item.get('status', {}).get('conditions', []):
if c.get('type') in ['Ready', 'Connected']:
print(f" [{c.get('type')}]: status={c.get('status')} reason={c.get('reason')} msg={c.get('message')}")

15
scripts/check_custom_coredns.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n kube-system get configmap coredns-custom")
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
client.close()

1
scripts/check_diff_kernel.py
View File

@@ -11,3 +11,4 @@ client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10
stdin, stdout, stderr = client.exec_command("cmp /boot/vmlinuz-6.12.109+rpt-rpi-2712 /boot/firmware/kernel_2712.img && echo IDENTICAL || echo DIFFERENT")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_dmesg_head.py
View File

@@ -11,3 +11,4 @@ client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10
stdin, stdout, stderr = client.exec_command("dmesg | head -n 10")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_dpkg_kernel.py
View File

@@ -11,3 +11,4 @@ client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10
stdin, stdout, stderr = client.exec_command("dpkg -l | grep -E 'linux-|raspberrypi-'")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_firmware_version.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("zcat /boot/firmware/kernel_2712.img 2>/dev/null | strings | grep 'Linux version' | head -n 5")
print("kernel_2712.img version:")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_fstab_ctl03.py
View File

@@ -21,3 +21,4 @@ for cmd in cmds:
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_getent.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("getent hosts aerodrome.snarfnet.net")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_harvester_ip.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("ping -c 2 harvester.spacedock.snarfnet.net; getent hosts harvester.spacedock.snarfnet.net")
print(stdout.read().decode('utf-8'))
client.close()

18
scripts/check_idrac_ips.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
for h in spacedock1.snarfnet.net spacedock2.snarfnet.net spacedock3.snarfnet.net spacedock4.snarfnet.net; do
getent hosts $h
done
""")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_kernel_file.py
View File

@@ -11,3 +11,4 @@ client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10
stdin, stdout, stderr = client.exec_command("file /boot/firmware/kernel_2712.img; ls -la /boot/vmlinuz*")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_kernel_str.py
View File

@@ -12,3 +12,4 @@ stdin, stdout, stderr = client.exec_command("strings /boot/firmware/kernel_2712.
print("Kernel version inside kernel_2712.img:")
print(stdout.read().decode('utf-8'))
client.close()

23
scripts/check_npm_certs.py Normal file
View File

@@ -0,0 +1,23 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
if out: print(out)
if err: print(f"ERR: {err}")
run("docker ps")
run("find / -name '*.crt' -o -name '*.pem' 2>/dev/null | grep -E 'live|letsencrypt|snarfnet' | head -n 30")
client.close()

15
scripts/check_npm_harvester.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("docker exec $(docker ps -q -f name=npm) grep -rnw '/etc/nginx/conf.d' -e 'harvester' 2>/dev/null; docker exec $(docker ps -q -f name=npm) grep -rn '/data/nginx/proxy_host' -e 'harvester' 2>/dev/null")
print("NPM config:")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_npm_hosts.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=npm) sqlite3 /data/database.sqlite "SELECT domain_names, forward_host, forward_port FROM proxy_host;"')
print("NPM hosts:")
print(stdout.read().decode('utf-8'))
client.close()

20
scripts/check_npm_live_certs.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
cmd = """
for cert in /data/glusterfs/swarm/brick/legacy_volumes/nginx/letsencrypt/live/npm-*/cert.pem; do
echo "=== $cert ==="
openssl x509 -in "$cert" -noout -subject -dates -ext subjectAltName 2>/dev/null
done
"""
stdin, stdout, stderr = client.exec_command(cmd)
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_npm_proxy_conf.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=nginx-proxy-manager_app) grep -r "proxy_pass" /data/nginx/proxy_host')
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_npm_real.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=nginx-proxy-manager_app) sqlite3 /data/database.sqlite "SELECT domain_names, forward_host, forward_port FROM proxy_host;"')
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_pkg_files.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("dpkg -L linux-image-6.12.109+rpt-rpi-2712 | grep -E '/boot|/lib/modules'")
print(stdout.read().decode('utf-8'))
client.close()

22
scripts/check_rancher_cluster_status.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
if out: print(out)
if err: print(f"ERR: {err}")
run("k3s kubectl get clusters.management.cattle.io")
client.close()

14
scripts/check_rancher_helm_values.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml /usr/local/bin/helm get values -n cattle-system rancher")
print("OUT:", stdout.read().decode('utf-8'))
print("ERR:", stderr.read().decode('utf-8'))
client.close()

14
scripts/check_rancher_ingress.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get ingress rancher -o yaml")
print(stdout.read().decode('utf-8'))
client.close()

13
scripts/check_rancher_status_final.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_real_uptime.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("uptime; who -b; last -x reboot | head -n 3; systemd-inhibit --list")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_resolv.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("cat /etc/resolv.conf")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_rpi_kernel.py
View File

@@ -23,3 +23,4 @@ for cmd in cmds:
print(stdout.read().decode('utf-8'))
client.close()

33
scripts/check_secret_dates.py Normal file
View File

@@ -0,0 +1,33 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
if out: print(out)
if err: print(f"ERR: {err}")
secrets = [
("cattle-system", "tls-rancher-ingress"),
("cattle-system", "tls-rancher"),
("cattle-system", "serving-cert"),
("default", "tls-authentik-cert"),
("monitoring", "zabbix-tls-cert"),
]
for ns, name in secrets:
print(f"\n--- Checking {ns}/{name} ---")
cmd = f"k3s kubectl -n {ns} get secret {name} -o jsonpath='{{.data.tls\\.crt}}' | base64 -d | openssl x509 -noout -subject -dates -issuer"
run(cmd)
client.close()

15
scripts/check_spacedock_agent.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.90", username="rancher", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('sudo kubectl -n cattle-system get pods -l app=cattle-cluster-agent')
print("=== SPACEDOCK AGENT PODS ===")
print(stdout.read().decode('utf-8'))
client.close()

20
scripts/check_spacedock_cluster.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
import yaml
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusters.management.cattle.io c-m-xgrxqfxh -o yaml")
data = yaml.safe_load(stdout.read().decode('utf-8'))
client.close()
print("Status:", data.get('status', {}).get('conditions', []))
print("Internal API endpoint:", data.get('status', {}).get('apiEndpoint'))
print("Driver:", data.get('status', {}).get('driver'))
print("Cluster agent image expected:", data.get('status', {}).get('agentImage'))

14
scripts/check_spacedock_detail.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('k3s kubectl get cluster.provisioning.cattle.io -n fleet-default spacedock -o jsonpath="{.status.conditions}"')
print("=== CONDITIONS ===")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/check_spaceport_agent_deploy.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get deployment cattle-cluster-agent -o yaml")
print(stdout.read().decode('utf-8'))
client.close()

25
scripts/check_spaceport_health.py
View File

@@ -1,16 +1,19 @@
import paramiko
import sys
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
sys.stdout.reconfigure(encoding='utf-8')
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
run("kubectl get applications -A -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
ssh.close()
stdin, stdout, stderr = client.exec_command('k3s kubectl get pods -n cattle-system')
print("=== SPACEPORT CATTLE-SYSTEM PODS ===")
print(stdout.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command('k3s kubectl get nodes')
print("=== SPACEPORT NODES ===")
print(stdout.read().decode('utf-8'))
client.close()

17
scripts/check_spaceport_nodes.py
View File

@@ -1,15 +1,14 @@
import os
import sys
import paramiko
import sys
sys.stdout.reconfigure(encoding="utf-8")
pkey = paramiko.Ed25519Key.from_private_key_file(os.path.expanduser('~/.ssh/id_ed25519_snarfnet'))
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", pkey=pkey, timeout=10)
stdin, stdout, stderr = client.exec_command("kubectl get nodes -o wide")
print("SPACEPORT KUBERNETES NODES:")
print(stdout.read().decode())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl get nodes -o wide")
print("=== SPACEPORT NODES ===")
print(stdout.read().decode('utf-8'))
client.close()

1
scripts/check_veth_file.py
View File

@@ -13,3 +13,4 @@ for node, ip in [('sninfr0lnxctl01', '140.44.4.41'), ('sninfr0lnxctl03', '140.44
stdin, stdout, stderr = client.exec_command("find /lib/modules -name '*veth*'; modinfo veth 2>&1")
print(stdout.read().decode('utf-8'))
client.close()

24
scripts/check_vmlinuz_version.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
python3 -c '
import gzip
with open("/boot/vmlinuz-6.12.109+rpt-rpi-2712", "rb") as f:
data = f.read()
# Search for Linux version string
import re
match = re.findall(b"Linux version [^\x00\n]+", data)
print("Found versions in vmlinuz:", match)
'
""")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_zcat.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("zcat /boot/vmlinuz-6.12.109+rpt-rpi-2712 2>/dev/null | strings | grep 'Linux version' | head -n 5")
print("zcat result:")
print(stdout.read().decode('utf-8'))
client.close()

13
scripts/del_test_pod.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
client.exec_command("k3s kubectl -n infrastructure delete pod semaphore-test --ignore-not-found")
client.close()

59
scripts/do_real_reboot.py Normal file
View File

@@ -0,0 +1,59 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
target_ip = "140.44.4.43"
print(f"Connecting to {target_ip} to issue systemctl reboot...")
client.connect(target_ip, username="root", key_filename=key_path, timeout=10)
try:
# Use systemctl reboot --force or nohup /sbin/reboot
stdin, stdout, stderr = client.exec_command("systemctl reboot")
print("reboot stdout:", stdout.read().decode('utf-8'))
print("reboot stderr:", stderr.read().decode('utf-8'))
except Exception as e:
print("Exception on reboot command (expected):", e)
client.close()
print("Waiting for node to drop off network...")
for i in range(30):
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(target_ip, username="root", key_filename=key_path, timeout=2)
c.close()
sys.stdout.write("x")
sys.stdout.flush()
time.sleep(2)
except Exception:
print("\nNode is DOWN! Reboot has taken effect!")
break
print("Waiting for node to boot back up...")
start = time.time()
up = False
while time.time() - start < 180:
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(target_ip, username="root", key_filename=key_path, timeout=3)
stdin, stdout, stderr = c.exec_command("uptime; uname -r")
out = stdout.read().decode('utf-8').strip()
c.close()
print(f"\nNode is UP!\n{out}")
up = True
break
except Exception:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(3)
if not up:
print("\nTimeout waiting for node.")

59
scripts/do_reboot_ctl02.py Normal file
View File

@@ -0,0 +1,59 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
target_ip = "140.44.4.42"
node_name = "sninfr0lnxctl02"
print(f"Connecting to {node_name} ({target_ip}) to issue reboot -f...")
client.connect(target_ip, username="root", key_filename=key_path, timeout=10)
try:
stdin, stdout, stderr = client.exec_command("/sbin/reboot -f")
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
except Exception as e:
print("Exception on reboot (expected):", e)
client.close()
print(f"Waiting for {node_name} to drop off network...")
for i in range(30):
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(target_ip, username="root", key_filename=key_path, timeout=2)
c.close()
sys.stdout.write("x")
sys.stdout.flush()
time.sleep(2)
except Exception:
print(f"\n{node_name} is DOWN! Reboot has taken effect!")
break
print(f"Waiting for {node_name} to boot back up...")
start = time.time()
up = False
while time.time() - start < 180:
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(target_ip, username="root", key_filename=key_path, timeout=3)
stdin, stdout, stderr = c.exec_command("uptime; uname -r; modprobe veth; lsmod | grep veth")
out = stdout.read().decode('utf-8').strip()
c.close()
print(f"\n{node_name} is UP!\n{out}")
up = True
break
except Exception:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(3)
if not up:
print(f"\nTimeout waiting for {node_name}.")

26
scripts/find_tls_certs.py Normal file
View File

@@ -0,0 +1,26 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
if out: print(out)
if err: print(f"ERR: {err}")
# Check all secrets of type kubernetes.io/tls in all namespaces
run("k3s kubectl get secrets -A --field-selector type=kubernetes.io/tls")
# Check cert-manager certificates if any
run("k3s kubectl get certificate -A")
run("k3s kubectl get clusterissuer,issuer -A")
client.close()

15
scripts/fix_coredns.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
client.exec_command("k3s kubectl -n kube-system delete configmap coredns-custom --ignore-not-found")
client.exec_command("k3s kubectl -n kube-system rollout restart deployment/coredns")
client.close()
print("Removed coredns-custom and restarted CoreDNS.")

16
scripts/get_import_cmd.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secret crt-token-default-token -n c-m-xgrxqfxh -o jsonpath='{.data.token}' | base64 -d")
token = stdout.read().decode('utf-8')
print("Token:", token)
print(f"Import command: curl --insecure -sfL https://aerodrome.snarfnet.net/v3/import/{token}_c-m-xgrxqfxh.yaml | kubectl apply -f -")
client.close()

14
scripts/get_reg_cmds.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusterregistrationtokens.management.cattle.io -o custom-columns=CLUSTER:.spec.clusterName,CMD:.status.command')
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/get_spacedock_token.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get clusterregistrationtokens.management.cattle.io -n c-m-xgrxqfxh -o yaml')
print(stdout.read().decode('utf-8'))
client.close()

25
scripts/inspect_rancher_certs.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
if out: print(out)
if err: print(f"ERR: {err}")
run("k3s kubectl get secret -n cattle-system tls-rancher-ingress -o yaml | grep -A 5 'tls.crt:'")
run("k3s kubectl get setting.management.cattle.io -o wide")
run("k3s kubectl get setting.management.cattle.io server-url -o yaml")
run("k3s kubectl get setting.management.cattle.io internal-server-url -o yaml")
client.close()

1
scripts/list_firmware.py
View File

@@ -11,3 +11,4 @@ client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10
stdin, stdout, stderr = client.exec_command("ls -la /boot/firmware")
print(stdout.read().decode('utf-8'))
client.close()

24
scripts/patch_hostaliases.py Normal file
View File

@@ -0,0 +1,24 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
patch_cmd = """k3s kubectl -n cattle-system patch deployment cattle-cluster-agent --type='strategic' -p '{"spec":{"template":{"spec":{"hostAliases":[{"ip":"140.44.4.34","hostnames":["aerodrome.snarfnet.net"]}]}}}}'"""
stdin, stdout, stderr = client.exec_command(patch_cmd)
print("PATCH STDOUT:", stdout.read().decode('utf-8'))
print("PATCH STDERR:", stderr.read().decode('utf-8'))
client.exec_command("k3s kubectl -n cattle-system rollout status deployment/cattle-cluster-agent --timeout=60s")
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=15")
print("\nLOGS AFTER PATCH:")
print(stdout.read().decode('utf-8'))
client.close()

14
scripts/ping_harvester.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("ping -c 2 140.44.4.90; curl -k -I https://140.44.4.90:443 | head -n 5")
print(stdout.read().decode('utf-8'))
client.close()

57
scripts/reboot_ctl02.py Normal file
View File

@@ -0,0 +1,57 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
target_ip = "140.44.4.42"
node_name = "sninfr0lnxctl02"
print(f"Connecting to {node_name} ({target_ip}) to issue systemctl reboot...")
client.connect(target_ip, username="root", key_filename=key_path, timeout=10)
try:
stdin, stdout, stderr = client.exec_command("systemctl reboot")
except Exception as e:
pass
client.close()
print(f"Waiting for {node_name} to drop off network...")
for i in range(30):
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(target_ip, username="root", key_filename=key_path, timeout=2)
c.close()
sys.stdout.write("x")
sys.stdout.flush()
time.sleep(2)
except Exception:
print(f"\n{node_name} is DOWN! Reboot has taken effect!")
break
print(f"Waiting for {node_name} to boot back up...")
start = time.time()
up = False
while time.time() - start < 180:
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(target_ip, username="root", key_filename=key_path, timeout=3)
stdin, stdout, stderr = c.exec_command("uptime; uname -r; modprobe veth; lsmod | grep veth")
out = stdout.read().decode('utf-8').strip()
c.close()
print(f"\n{node_name} is UP!\n{out}")
up = True
break
except Exception:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(3)
if not up:
print(f"\nTimeout waiting for {node_name}.")

27
scripts/restart_agent.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system rollout restart deployment cattle-cluster-agent")
print(stdout.read().decode('utf-8'))
time.sleep(10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide")
print("=== PODS ===")
print(stdout.read().decode('utf-8'))
time.sleep(5)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=30")
print("=== AGENT LOGS ===")
print(stdout.read().decode('utf-8'))
client.close()

20
scripts/sweep_spaceport.py Normal file
View File

@@ -0,0 +1,20 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl get pods -A | grep -v -E 'Running|Completed'")
print("Non-running/non-completed pods across Spaceport:")
out = stdout.read().decode('utf-8').strip()
if out:
print(out)
else:
print("ALL PODS ACROSS ALL NAMESPACES ARE RUNNING OR COMPLETED! (100% HEALTHY)")
client.close()

29
scripts/test_dns_resolution.py Normal file
View File

@@ -0,0 +1,29 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
cmds = [
"nslookup aerodrome.snarfnet.net",
"curl -k -I https://174.106.252.96:443 -H 'Host: aerodrome.snarfnet.net' --max-time 5",
"curl -k -I https://140.44.4.34:443 -H 'Host: aerodrome.snarfnet.net' --max-time 5"
]
for cmd in cmds:
print(f"\n>>> Running on Spaceport: {cmd}")
stdin, stdout, stderr = client.exec_command(cmd)
out = stdout.read().decode('utf-8').strip()
err = stderr.read().decode('utf-8').strip()
if out:
print(out)
if err:
print("STDERR:", err)
client.close()

21
scripts/test_harvester_ssh.py Normal file
View File

@@ -0,0 +1,21 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for u in ['rancher', 'root']:
try:
print(f"Connecting to 140.44.4.90 as {u}...")
client.connect("140.44.4.90", username=u, key_filename=key_path, timeout=5)
print(f"Connected as {u}!")
stdin, stdout, stderr = client.exec_command("sudo kubectl get pods -n cattle-system")
print(stdout.read().decode('utf-8'))
client.close()
break
except Exception as e:
print(f"Failed as {u}: {e}")

16
scripts/test_served_cert.py Normal file
View File

@@ -0,0 +1,16 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
cmd = "echo | openssl s_client -connect 140.44.4.34:443 -servername aerodrome.snarfnet.net 2>/dev/null | openssl x509 -noout -subject -dates -issuer"
stdin, stdout, stderr = client.exec_command(cmd)
print("Served cert by Traefik on 140.44.4.34:443:")
print(stdout.read().decode('utf-8'))
client.close()

56
scripts/update_rancher_cert.py Normal file
View File

@@ -0,0 +1,56 @@
import paramiko
import sys
import base64
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
# 1. Read cert and key from 140.44.4.71
c_npm = paramiko.SSHClient()
c_npm.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c_npm.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = c_npm.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /etc/letsencrypt/live/npm-8/fullchain.pem")
fullchain = stdout.read()
stdin, stdout, stderr = c_npm.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /etc/letsencrypt/live/npm-8/privkey.pem")
privkey = stdout.read()
c_npm.close()
if not fullchain or not privkey:
print("Error reading cert or key from NPM!")
sys.exit(1)
print(f"Read fullchain ({len(fullchain)} bytes) and privkey ({len(privkey)} bytes) successfully.")
# 2. Connect to 140.44.4.11 (Aerodrome master) and update secret
c_aero = paramiko.SSHClient()
c_aero.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c_aero.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
b64_fullchain = base64.b64encode(fullchain).decode('ascii')
b64_privkey = base64.b64encode(privkey).decode('ascii')
cmd_write = f"""
echo '{b64_fullchain}' | base64 -d > /tmp/aerodrome_fullchain.pem
echo '{b64_privkey}' | base64 -d > /tmp/aerodrome_privkey.pem
k3s kubectl -n cattle-system create secret tls tls-rancher-ingress \
--cert=/tmp/aerodrome_fullchain.pem \
--key=/tmp/aerodrome_privkey.pem \
--dry-run=client -o yaml | k3s kubectl apply -f -
rm -f /tmp/aerodrome_fullchain.pem /tmp/aerodrome_privkey.pem
"""
stdin, stdout, stderr = c_aero.exec_command(cmd_write)
print("kubectl apply result:")
print(stdout.read().decode('utf-8'))
err = stderr.read().decode('utf-8')
if err: print("kubectl stderr:", err)
# Verify updated secret
stdin, stdout, stderr = c_aero.exec_command("k3s kubectl -n cattle-system get secret tls-rancher-ingress -o jsonpath='{.data.tls\\.crt}' | base64 -d | openssl x509 -noout -subject -dates -issuer")
print("\nNew secret details:")
print(stdout.read().decode('utf-8'))
c_aero.close()

15
scripts/verify_npm1.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) openssl x509 -in /etc/letsencrypt/live/npm-1/fullchain.pem -noout -subject -dates -issuer")
print("npm-1 x509 details:")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/verify_npm8.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app) openssl x509 -in /etc/letsencrypt/live/npm-8/fullchain.pem -noout -subject -dates -issuer")
print("npm-8 x509 details:")
print(stdout.read().decode('utf-8'))
client.close()