diff --git a/homelab-gitops-repo b/homelab-gitops-repo index cb8e60b..c532600 160000 --- a/homelab-gitops-repo +++ b/homelab-gitops-repo @@ -1 +1 @@ -Subproject commit cb8e60b086646e225edc3bd85dc6868033fb6ab1 +Subproject commit c532600727449064cc4ca73b20d900e218f3c2ad diff --git a/scripts/audit_cluster_workloads.py b/scripts/audit_cluster_workloads.py new file mode 100644 index 0000000..41cca67 --- /dev/null +++ b/scripts/audit_cluster_workloads.py @@ -0,0 +1,21 @@ +import paramiko + +def inspect_cluster(ip, name): + print(f"\n=======================================================") + print(f"=== {name} WORKLOADS ({ip}) ===") + print(f"=======================================================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + cmd = "kubectl get deploy,sts,ds -A -o custom-columns=NAMESPACE:.metadata.namespace,KIND:.kind,NAME:.metadata.name,IMAGES:.spec.template.spec.containers[*].image" + _, out, _ = ssh.exec_command(cmd) + lines = out.read().decode().splitlines() + for l in lines: + if not any(k in l for k in ['kube-system', 'cattle-', 'metallb-system', 'longhorn-system']): + print(l) + ssh.close() + +inspect_cluster('140.44.4.11', 'AERODROME') +inspect_cluster('140.44.4.41', 'SPACEPORT') + diff --git a/scripts/audit_gitops_manifests.py b/scripts/audit_gitops_manifests.py new file mode 100644 index 0000000..5042a56 --- /dev/null +++ b/scripts/audit_gitops_manifests.py @@ -0,0 +1,40 @@ +import os +import re +import paramiko + +manifest_root = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests" + +print("=== GITOPS MANIFEST IMAGE AUDIT ===") + +image_re = re.compile(r'image:\s*([^\s"\']+)', re.IGNORECASE) +chart_re = re.compile(r'chart:\s*([^\s"\']+)', re.IGNORECASE) +version_re = re.compile(r'version:\s*([^\s"\']+)', re.IGNORECASE) + +manifest_data = {} + +for root, dirs, files in os.walk(manifest_root): + for f in files: + if f.endswith('.yaml') or f.endswith('.yml'): + path = os.path.join(root, f) + rel_path = os.path.relpath(path, manifest_root) + with open(path, 'r', encoding='utf-8', errors='ignore') as fp: + content = fp.read() + images = image_re.findall(content) + charts = chart_re.findall(content) + versions = version_re.findall(content) + if images or charts: + manifest_data[rel_path] = { + 'images': list(set(images)), + 'charts': list(set(charts)), + 'versions': list(set(versions)) + } + +for rel_path, data in sorted(manifest_data.items()): + print(f"\nFile: {rel_path}") + if data['images']: + print(" Images:", ", ".join(data['images'])) + if data['charts']: + print(" Charts:", ", ".join(data['charts'])) + if data['versions']: + print(" Versions:", ", ".join(data['versions'])) + diff --git a/scripts/check_51.py b/scripts/check_51.py new file mode 100644 index 0000000..6c45cb5 --- /dev/null +++ b/scripts/check_51.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("ss -tulpn | grep ':80 '") +run("ip a") +run("kubectl get pod -A -o wide | grep sninfr4lnxwrk01 | grep pihole || true") +ssh.close() + diff --git a/scripts/check_51_nat.py b/scripts/check_51_nat.py new file mode 100644 index 0000000..754054c --- /dev/null +++ b/scripts/check_51_nat.py @@ -0,0 +1,18 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("iptables -t nat -S | grep -E '80\b|31905|31937'") +run("crictl ps") +ssh.close() + diff --git a/scripts/check_aerodrome_pods.py b/scripts/check_aerodrome_pods.py new file mode 100644 index 0000000..a371627 --- /dev/null +++ b/scripts/check_aerodrome_pods.py @@ -0,0 +1,15 @@ +import paramiko + +s = paramiko.SSHClient() +s.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, _ = s.exec_command(cmd) + print(out.read().decode()) + +run("kubectl get pods -A | grep -v -E 'Running|Completed' || true") +run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state") +s.close() + diff --git a/scripts/check_aerodrome_preflight.py b/scripts/check_aerodrome_preflight.py new file mode 100644 index 0000000..65e933b --- /dev/null +++ b/scripts/check_aerodrome_preflight.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("kubectl get nodes") +run("kubectl get pods -A | grep -v -E 'Running|Completed' || true") +run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness | grep -v healthy || true") +ssh.close() + diff --git a/scripts/check_aerodrome_quick.py b/scripts/check_aerodrome_quick.py new file mode 100644 index 0000000..3b94114 --- /dev/null +++ b/scripts/check_aerodrome_quick.py @@ -0,0 +1,8 @@ +import paramiko +ssh=paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +_,o,_=ssh.exec_command('kubectl get nodes') +print(o.read().decode()) +ssh.close() + diff --git a/scripts/check_all_coder_snapshots.py b/scripts/check_all_coder_snapshots.py new file mode 100644 index 0000000..7ab93d0 --- /dev/null +++ b/scripts/check_all_coder_snapshots.py @@ -0,0 +1,24 @@ +import paramiko + +nodes = [ + ("wrk01", "140.44.4.51"), + ("wrk02", "140.44.4.52"), + ("wrk03", "140.44.4.53"), + ("wrk04", "140.44.4.54"), + ("wrk05", "140.44.4.55"), + ("wrk06", "140.44.4.56"), +] + +for name, ip in nodes: + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=4) + _, out, _ = ssh.exec_command("find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path '*/home/coder/.config/coderv2' 2>/dev/null") + res = out.read().decode().strip() + if res: + print(f"[{name} - {ip}] Found:\n{res}") + ssh.close() + except Exception as e: + print(f"[{name}] err: {e}") + diff --git a/scripts/check_all_endpoints.py b/scripts/check_all_endpoints.py new file mode 100644 index 0000000..665fd8a --- /dev/null +++ b/scripts/check_all_endpoints.py @@ -0,0 +1,19 @@ +import subprocess + +endpoints = [ + ("Argo CD", "https://argo.snarfnet.net"), + ("Rancher Manager", "https://aerodrome.snarfnet.net"), + ("AWX Console", "https://awx.snarfnet.net"), + ("Authentik SSO", "https://authentik.snarfnet.net"), + ("Uptime Kuma", "https://uptime.snarfnet.net/dashboard"), + ("Gitea GitOps", "http://140.44.4.51:90"), + ("Zabbix Server", "http://140.44.4.51:8888"), + ("Portainer Server", "https://140.44.4.71:9443") +] + +print("=== ENDPOINT HEALTH CHECK ===") +for name, url in endpoints: + res = subprocess.run(f"curl.exe -k -s -o nul -w '%{{http_code}}' {url}", capture_output=True, text=True, shell=True) + code = res.stdout.strip() + print(f"[{code}] {name} ({url})") + diff --git a/scripts/check_argo_apps.py b/scripts/check_argo_apps.py new file mode 100644 index 0000000..3f78326 --- /dev/null +++ b/scripts/check_argo_apps.py @@ -0,0 +1,14 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +stdin, stdout, stderr = ssh.exec_command("kubectl get applications -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status") +print(stdout.read().decode()) +err = stderr.read().decode() +if err: + print("ERR:", err) + +ssh.close() + diff --git a/scripts/check_argo_ing.py b/scripts/check_argo_ing.py new file mode 100644 index 0000000..eca1992 --- /dev/null +++ b/scripts/check_argo_ing.py @@ -0,0 +1,22 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("kubectl get ingress argo-ing -n infrastructure -o yaml") +run("kubectl get svc pihole-web -n infrastructure -o yaml") +run("kubectl get svc -n infrastructure") +run("kubectl get svc -A | grep -E 'LoadBalancer|140.44.4.62'") +run("curl -k -I -H 'Host: argo.snarfnet.net' http://140.44.4.62") +run("curl -k -I -H 'Host: argo.snarfnet.net' https://140.44.4.62") +ssh.close() + diff --git a/scripts/check_argocd_now.py b/scripts/check_argocd_now.py new file mode 100644 index 0000000..190bca4 --- /dev/null +++ b/scripts/check_argocd_now.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command("kubectl get applications -n infrastructure -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status") +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/check_cattle_pods.py b/scripts/check_cattle_pods.py new file mode 100644 index 0000000..9d393bf --- /dev/null +++ b/scripts/check_cattle_pods.py @@ -0,0 +1,14 @@ +import paramiko + +def check_cattle(ip, cluster): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print(f"\n[{cluster}] Checking cattle-system pods...") + _, out, _ = ssh.exec_command("kubectl get pods -n cattle-system -o wide") + print(out.read().decode().strip()) + ssh.close() + +check_cattle('140.44.4.11', 'AERODROME') +check_cattle('140.44.4.41', 'SPACEPORT') + diff --git a/scripts/check_chart_versions.py b/scripts/check_chart_versions.py new file mode 100644 index 0000000..7d94e1a --- /dev/null +++ b/scripts/check_chart_versions.py @@ -0,0 +1,19 @@ +import paramiko + +def check_catalog_versions(ip, cluster_name): + print(f"\n=======================================================") + print(f"=== {cluster_name} AVAILABLE CHART VERSIONS ===") + print(f"=======================================================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + # We can inspect the Rancher catalog charts + cmd = "kubectl get charts.catalog.cattle.io -A -o custom-columns=REPO:.metadata.namespace,CHART:.metadata.name | head -n 30" + _, out, _ = ssh.exec_command(cmd) + print(out.read().decode()) + ssh.close() + +check_catalog_versions('140.44.4.11', 'AERODROME') +check_catalog_versions('140.44.4.41', 'SPACEPORT') + diff --git a/scripts/check_coder_logs.py b/scripts/check_coder_logs.py new file mode 100644 index 0000000..b7a4f11 --- /dev/null +++ b/scripts/check_coder_logs.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command('kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=30') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/check_coder_page.py b/scripts/check_coder_page.py new file mode 100644 index 0000000..829e189 --- /dev/null +++ b/scripts/check_coder_page.py @@ -0,0 +1,14 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command("curl -s https://coder.snarfnet.net/api/v2/users/first") +print("First user status:", out.read().decode()) + +_, out2, _ = ssh.exec_command("curl -s https://coder.snarfnet.net/api/v2/experiments") +print("Experiments:", out2.read().decode()) + +ssh.close() + diff --git a/scripts/check_ghcr_tags.py b/scripts/check_ghcr_tags.py new file mode 100644 index 0000000..290279f --- /dev/null +++ b/scripts/check_ghcr_tags.py @@ -0,0 +1,24 @@ +import urllib.request +import json + +def get_ghcr_tags(package): + # Public unauthenticated ghcr tags endpoint + url = f"https://ghcr.io/v2/{package}/tags/list" + try: + # Get anonymous token + token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{package}:pull" + with urllib.request.urlopen(token_url, timeout=5) as t_resp: + token = json.loads(t_resp.read().decode())['token'] + + req = urllib.request.Request(url, headers={'Authorization': f'Bearer {token}'}) + with urllib.request.urlopen(req, timeout=5) as resp: + data = json.loads(resp.read().decode()) + tags = [t for t in data.get('tags', []) if not t.endswith('-dev') and not 'beta' in t] + return tags[-10:] + except Exception as e: + return [f"Err: {e}"] + +print("Homepage tags:", get_ghcr_tags("gethomepage/homepage")) +print("Coder tags:", get_ghcr_tags("coder/coder")) +print("Authentik tags:", get_ghcr_tags("goauthentik/server")) + diff --git a/scripts/check_grafana_spec.py b/scripts/check_grafana_spec.py new file mode 100644 index 0000000..f3b1e33 --- /dev/null +++ b/scripts/check_grafana_spec.py @@ -0,0 +1,15 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, _ = ssh.exec_command(cmd) + print(out.read().decode()) + +run("kubectl get deployment grafana -n default -o jsonpath='{.spec.template.spec.containers[*].name}'") +run("kubectl get deployment grafana -n default -o jsonpath='{.spec.template.spec.containers[*].image}'") +ssh.close() + diff --git a/scripts/check_helm_releases.py b/scripts/check_helm_releases.py new file mode 100644 index 0000000..02fb780 --- /dev/null +++ b/scripts/check_helm_releases.py @@ -0,0 +1,14 @@ +import paramiko + +def check_helm(ip, name): + print(f"\n=== {name} HELM RELEASES ({ip}) ===") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + _, out, _ = ssh.exec_command("helm list -A") + print(out.read().decode()) + ssh.close() + +check_helm('140.44.4.11', 'AERODROME') +check_helm('140.44.4.41', 'SPACEPORT') + diff --git a/scripts/check_historical_data.py b/scripts/check_historical_data.py new file mode 100644 index 0000000..c091122 --- /dev/null +++ b/scripts/check_historical_data.py @@ -0,0 +1,60 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +# 1. Inspect Gotify database on Swarm +print("=================== GOTIFY (Swarm 140.44.4.71) ===================") +ssh_swm = paramiko.SSHClient() +ssh_swm.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_swm.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run_swm(cmd): + _, out, err = ssh_swm.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +run_swm("cat c:\\Users\\Snarf\\OneDrive\\Desktop\\code\\SnarfCode\\docker-deployments\\gotify.yml 2>/dev/null || true") +run_swm("sudo docker service inspect gotify_gotify --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'") +run_swm("sudo docker run --rm -v gotify_gotify_data:/data alpine ls -la /data 2>/dev/null || find /var/lib/docker/volumes/ -name 'gotify.db' 2>/dev/null") + +# Query gotify.db if found +run_swm("""python3 -c ' +import sqlite3, glob +db_files = glob.glob("/var/lib/docker/volumes/**/gotify.db", recursive=True) +print("Gotify DB files:", db_files) +for db in db_files: + try: + conn = sqlite3.connect(db) + cur = conn.cursor() + print("Tables in", db) + for row in cur.execute("SELECT name FROM sqlite_master WHERE type=\"table\""): + print(" ", row[0]) + print("Applications count:", cur.execute("SELECT count(*) FROM applications").fetchone()[0]) + print("Clients count:", cur.execute("SELECT count(*) FROM clients").fetchone()[0]) + print("Messages count:", cur.execute("SELECT count(*) FROM messages").fetchone()[0]) + except Exception as e: + print("Err reading", db, e) +'""") + +ssh_swm.close() + +# 2. Inspect Semaphore BoltDB on Spaceport +print("\n=================== SEMAPHORE (Spaceport 140.44.4.41) ===================") +ssh_sp = paramiko.SSHClient() +ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run_sp(cmd): + _, out, err = ssh_sp.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +run_sp("kubectl exec -n infrastructure deployment/semaphore -- ls -lh /var/lib/semaphore/database.boltdb") +# Let's inspect projects and templates using semaphore CLI or strings +run_sp("kubectl exec -n infrastructure deployment/semaphore -- strings /var/lib/semaphore/database.boltdb | grep -E 'project|playbook|inventory|repository' | head -n 30") + +ssh_sp.close() + diff --git a/scripts/check_homepage_spec.py b/scripts/check_homepage_spec.py new file mode 100644 index 0000000..89d54ca --- /dev/null +++ b/scripts/check_homepage_spec.py @@ -0,0 +1,15 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, _ = ssh.exec_command(cmd) + print(out.read().decode()) + +run("kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath='{.spec.template.spec.containers[*].name}'") +run("kubectl get deployment homepage-2-1744068158 -n infrastructure -o jsonpath='{.spec.template.spec.containers[*].image}'") +ssh.close() + diff --git a/scripts/check_jenkins.py b/scripts/check_jenkins.py new file mode 100644 index 0000000..7ce2138 --- /dev/null +++ b/scripts/check_jenkins.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +run("kubectl get pod -n infrastructure jenkins-0 -o wide") +run("kubectl describe pod -n infrastructure jenkins-0 | tail -n 25") +run("curl -k -s -I https://jenkins.snarfnet.net | head -n 5") +run("curl -s -I http://140.44.4.52:101 | head -n 5") + +ssh.close() + diff --git a/scripts/check_job134.py b/scripts/check_job134.py new file mode 100644 index 0000000..e2b1cb2 --- /dev/null +++ b/scripts/check_job134.py @@ -0,0 +1,28 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=134) +print(f"Job [134]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}") +events = job.job_events.filter(event__in=['runner_on_ok', 'runner_on_failed', 'playbook_on_stats']).order_by('-id')[:10] +for e in events: + print(f"[{e.event}] Host: {e.event_data.get('host')} - Task: {e.event_data.get('task')}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) + +_, out, _ = ssh.exec_command('kubectl get nodes') +print("=== KUBERNETES NODES ===") +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/check_job134_end.py b/scripts/check_job134_end.py new file mode 100644 index 0000000..56a5d58 --- /dev/null +++ b/scripts/check_job134_end.py @@ -0,0 +1,20 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=134) +print(job.result_stdout[-1500:]) +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/check_job134_explain.py b/scripts/check_job134_explain.py new file mode 100644 index 0000000..aee5477 --- /dev/null +++ b/scripts/check_job134_explain.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=134) +print(f"Status: {job.status}") +print(f"Job explanation: {job.job_explanation}") +print(f"Result Traceback: {job.result_traceback}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/check_job134_fail.py b/scripts/check_job134_fail.py new file mode 100644 index 0000000..214b0ad --- /dev/null +++ b/scripts/check_job134_fail.py @@ -0,0 +1,25 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=134) +failed_events = job.job_events.filter(failed=True) +for e in failed_events: + res = e.event_data.get('res', {}) + msg = res.get('msg') or res.get('stderr') or res.get('stdout') or str(res) + print(f"Host: {e.event_data.get('host')}, Task: {e.event_data.get('task')}") + print(f"Details: {msg[:300]}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/check_job134_stdout.py b/scripts/check_job134_stdout.py new file mode 100644 index 0000000..9a75c37 --- /dev/null +++ b/scripts/check_job134_stdout.py @@ -0,0 +1,21 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=134) +lines = [l for l in job.result_stdout.split('\\n') if l.strip()] +print('\\n'.join(lines[-35:])) +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/check_job_status.py b/scripts/check_job_status.py new file mode 100644 index 0000000..2a2c17e --- /dev/null +++ b/scripts/check_job_status.py @@ -0,0 +1,21 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=131) +print(f"Job [131]: Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/check_k3s_channels.py b/scripts/check_k3s_channels.py new file mode 100644 index 0000000..41e6814 --- /dev/null +++ b/scripts/check_k3s_channels.py @@ -0,0 +1,12 @@ +import urllib.request +import json + +url = "https://update.k3s.io/v1-release/channels" +try: + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json'}) + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode()) + for item in data.get('data', []): + print(f"{item.get('id')}: {item.get('latest')}") +except Exception as e: + print("Error:", e) diff --git a/scripts/check_logs_utf8.py b/scripts/check_logs_utf8.py new file mode 100644 index 0000000..ad2805a --- /dev/null +++ b/scripts/check_logs_utf8.py @@ -0,0 +1,32 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh_sp = paramiko.SSHClient() +ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +ssh_swm = paramiko.SSHClient() +ssh_swm.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_swm.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(ssh, cmd): + print(f"\n--- {cmd} ---") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +print("=== PEANUT LOGS (Swarm) ===") +run(ssh_swm, "sudo docker service logs peanut_peanut --tail=50") + +print("\n=== CODER LOGS (Spaceport) ===") +run(ssh_sp, "kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=50") + +print("\n=== SEMAPHORE LOGS (Spaceport) ===") +run(ssh_sp, "kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=50") + +ssh_sp.close() +ssh_swm.close() + diff --git a/scripts/check_longhorn_version.py b/scripts/check_longhorn_version.py new file mode 100644 index 0000000..180c980 --- /dev/null +++ b/scripts/check_longhorn_version.py @@ -0,0 +1,15 @@ +import paramiko + +def check_lh(ip, cluster): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print(f"\n[{cluster}] Checking Longhorn images...") + _, out, _ = ssh.exec_command("kubectl get daemonset -n longhorn-system longhorn-manager -o jsonpath='{.spec.template.spec.containers[0].image}{\"\\n\"}'") + print("Manager image:", out.read().decode().strip()) + _, out, _ = ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state | head -n 10") + print(out.read().decode().strip()) + ssh.close() + +check_lh('140.44.4.11', 'AERODROME') +check_lh('140.44.4.41', 'SPACEPORT') diff --git a/scripts/check_npm_all_hosts.py b/scripts/check_npm_all_hosts.py new file mode 100644 index 0000000..73df45a --- /dev/null +++ b/scripts/check_npm_all_hosts.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +stdin, stdout, stderr = ssh.exec_command("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -E "server_name|set \\$server|set \\$port" /data/nginx/proxy_host/*.conf""") +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/check_npm_argo.py b/scripts/check_npm_argo.py new file mode 100644 index 0000000..d52a549 --- /dev/null +++ b/scripts/check_npm_argo.py @@ -0,0 +1,21 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("sudo docker ps | grep nginx") +run("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') find /data -maxdepth 3") +run("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') cat /etc/hosts") +run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -rn "argo" /etc/nginx/ /data/nginx/""") +run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') grep -rn "pihole" /etc/nginx/ /data/nginx/""") + +ssh.close() diff --git a/scripts/check_npm_db.py b/scripts/check_npm_db.py new file mode 100644 index 0000000..023eb2e --- /dev/null +++ b/scripts/check_npm_db.py @@ -0,0 +1,15 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +cmd = "sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c \"python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print('\\n'.join(str(r) for r in cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host WHERE is_deleted=0')))\\\"\"" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +err = stderr.read().decode() +if err: + print("ERR:", err) + +ssh.close() + diff --git a/scripts/check_npm_mounts.py b/scripts/check_npm_mounts.py new file mode 100644 index 0000000..b2329a8 --- /dev/null +++ b/scripts/check_npm_mounts.py @@ -0,0 +1,15 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + +run("sudo docker service inspect nginx-proxy-manager_app --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'") +run("sudo docker inspect $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') --format '{{json .Mounts}}'") +ssh.close() + diff --git a/scripts/check_npm_nodes.py b/scripts/check_npm_nodes.py new file mode 100644 index 0000000..2f8b617 --- /dev/null +++ b/scripts/check_npm_nodes.py @@ -0,0 +1,14 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + +run("sudo docker service ps nginx-proxy-manager_app --filter 'desired-state=running'") +ssh.close() + diff --git a/scripts/check_npm_user.py b/scripts/check_npm_user.py new file mode 100644 index 0000000..fa45bde --- /dev/null +++ b/scripts/check_npm_user.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +# Check user table in NPM db +run("""sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c "python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); print(list(cur.execute('SELECT id, name, email FROM user')))\\\"\"""") + +ssh.close() + diff --git a/scripts/check_old_coder_layer.py b/scripts/check_old_coder_layer.py new file mode 100644 index 0000000..061ec02 --- /dev/null +++ b/scripts/check_old_coder_layer.py @@ -0,0 +1,26 @@ +import paramiko + +nodes = [ + ("ctl01", "140.44.4.41"), + ("wrk01", "140.44.4.51"), + ("wrk02", "140.44.4.52"), + ("wrk03", "140.44.4.53"), + ("wrk04", "140.44.4.54"), + ("wrk05", "140.44.4.55"), + ("wrk06", "140.44.4.56"), +] + +for name, ip in nodes: + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5) + _, out, _ = ssh.exec_command("find /run/k3s/containerd/ /var/lib/rancher/k3s/agent/containerd/ -type d -name 'coderv2' 2>/dev/null") + res = out.read().decode().strip() + if res: + print(f"[{name} - {ip}] FOUND coderv2:\n{res}") + else: + print(f"[{name} - {ip}] none") + ssh.close() + except Exception as e: + print(f"[{name} - {ip}] err: {e}") diff --git a/scripts/check_other_apps.py b/scripts/check_other_apps.py new file mode 100644 index 0000000..8a46db3 --- /dev/null +++ b/scripts/check_other_apps.py @@ -0,0 +1,22 @@ +import paramiko + +def query_cluster(ip, name): + print(f"\n=================== CLUSTER {name} ({ip}) ===================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + + run("kubectl get ns") + run("kubectl get deploy,sts -A | grep -E 'awx|authentik|zabbix|coder'") + ssh.close() + +query_cluster('140.44.4.11', 'AERODROME') +query_cluster('140.44.4.41', 'SPACEPORT') + diff --git a/scripts/check_peanut_now.py b/scripts/check_peanut_now.py new file mode 100644 index 0000000..1d7e97d --- /dev/null +++ b/scripts/check_peanut_now.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +run("sudo docker service logs peanut_peanut --tail=25") +run("curl -s -I http://140.44.4.70:8050/") +run("curl -s http://140.44.4.70:8050/ | grep -i setup || echo 'NO SETUP REDIRECT'") + +ssh.close() + diff --git a/scripts/check_peanut_storage.py b/scripts/check_peanut_storage.py new file mode 100644 index 0000000..1b5c2f6 --- /dev/null +++ b/scripts/check_peanut_storage.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n--- {cmd} ---") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +run("ls -la /mnt/swarm_shared/legacy_volumes/peanut/") +run("ssh snshpd4lnxswm05 'ls -la /mnt/swarm_shared/legacy_volumes/peanut/'") +run("sudo docker service inspect peanut_peanut") + +ssh.close() + diff --git a/scripts/check_pvc_progress.py b/scripts/check_pvc_progress.py new file mode 100644 index 0000000..08d27a4 --- /dev/null +++ b/scripts/check_pvc_progress.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command('kubectl get pods,pvc -n infrastructure | grep coder') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/check_rancher_compat.py b/scripts/check_rancher_compat.py new file mode 100644 index 0000000..90bcacf --- /dev/null +++ b/scripts/check_rancher_compat.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("kubectl get settings.management.cattle.io server-version -o yaml") +run("kubectl get pod -n cattle-system -l app=rancher -o jsonpath='{.items[0].spec.containers[*].image}'") +run("kubectl get clusters.management.cattle.io -o custom-columns=NAME:.metadata.name,DISPLAY:.spec.displayName,VERSION:.status.version.gitVersion") +ssh.close() + diff --git a/scripts/check_removal_script.py b/scripts/check_removal_script.py new file mode 100644 index 0000000..885caa4 --- /dev/null +++ b/scripts/check_removal_script.py @@ -0,0 +1,17 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('cat /proc/3712358/cmdline') +print("CMDLINE:", stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('cat /usr/sbin/linux-check-removal || cat /usr/bin/linux-check-removal') +print("SCRIPT:\n", stdout.read().decode()) + +ssh.close() + diff --git a/scripts/check_semaphore_crash.py b/scripts/check_semaphore_crash.py new file mode 100644 index 0000000..56c2cd1 --- /dev/null +++ b/scripts/check_semaphore_crash.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + errors = err.read().decode().strip() + if res: + print(res) + if errors: + print("ERR:", errors) + +run("kubectl logs deployment/semaphore -n infrastructure --tail=40") +ssh.close() + diff --git a/scripts/check_semaphore_help.py b/scripts/check_semaphore_help.py new file mode 100644 index 0000000..29ef392 --- /dev/null +++ b/scripts/check_semaphore_help.py @@ -0,0 +1,14 @@ +import paramiko +import time + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +ssh.exec_command('kubectl run semaphore-test --image=semaphoreui/semaphore:v2.19.17 --restart=Never -n infrastructure --command -- strings /usr/bin/semaphore') +time.sleep(6) +_, out, _ = ssh.exec_command('kubectl logs semaphore-test -n infrastructure | grep -E "mysql|postgres|bolt" | head -n 30') +print(out.read().decode()) +ssh.exec_command('kubectl delete pod semaphore-test -n infrastructure --grace-period=0 --force') +ssh.close() + diff --git a/scripts/check_semaphore_stable.py b/scripts/check_semaphore_stable.py new file mode 100644 index 0000000..68b9eb2 --- /dev/null +++ b/scripts/check_semaphore_stable.py @@ -0,0 +1,9 @@ +import urllib.request +import json + +url = "https://hub.docker.com/v2/repositories/semaphoreui/semaphore/tags?page_size=100" +req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) +with urllib.request.urlopen(req, timeout=5) as resp: + data = json.loads(resp.read().decode()) + ga_tags = [t['name'] for t in data.get('results', []) if not any(x in t['name'] for x in ['-rc', '-beta', '-alpha', '-debug', 'latest'])] + print("Semaphore GA tags:", ga_tags[:10]) diff --git a/scripts/check_spaceport_health.py b/scripts/check_spaceport_health.py new file mode 100644 index 0000000..fdd0e3b --- /dev/null +++ b/scripts/check_spaceport_health.py @@ -0,0 +1,16 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + +run("kubectl get pods -A | grep -v -E 'Running|Completed' || true") +run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state") +run("kubectl get applications -A -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status") +ssh.close() + diff --git a/scripts/check_spaceport_lb.py b/scripts/check_spaceport_lb.py new file mode 100644 index 0000000..8943db7 --- /dev/null +++ b/scripts/check_spaceport_lb.py @@ -0,0 +1,18 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + stdin, stdout, stderr = ssh.exec_command(cmd) + print(stdout.read().decode()) + err = stderr.read().decode() + if err: + print("ERR:", err) + +run("kubectl get ds -A") +run("kubectl get svc traefik -n kube-system -o yaml") +ssh.close() + diff --git a/scripts/check_spaceport_packages.py b/scripts/check_spaceport_packages.py new file mode 100644 index 0000000..d5f44f2 --- /dev/null +++ b/scripts/check_spaceport_packages.py @@ -0,0 +1,31 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +spaceport_nodes = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44'), + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56') +] + +for name, ip in spaceport_nodes: + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=5) + stdin, stdout, stderr = ssh.exec_command('apt list --upgradable 2>/dev/null | grep -v "Listing" | wc -l') + cnt = stdout.read().decode().strip() + print(f"[{name}] ({ip}): {cnt} packages upgradeable") + ssh.close() + except Exception as e: + print(f"[{name}] ({ip}): Error connecting: {e}") + diff --git a/scripts/check_spaceport_quick.py b/scripts/check_spaceport_quick.py new file mode 100644 index 0000000..a486e50 --- /dev/null +++ b/scripts/check_spaceport_quick.py @@ -0,0 +1,9 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) +_, out, _ = ssh.exec_command('kubectl get nodes') +print(out.read().decode()) +ssh.close() + diff --git a/scripts/check_upstream_tags.py b/scripts/check_upstream_tags.py new file mode 100644 index 0000000..0e1c470 --- /dev/null +++ b/scripts/check_upstream_tags.py @@ -0,0 +1,18 @@ +import urllib.request +import json + +def get_dockerhub_latest(repo, prefix=""): + url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=15" + try: + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, timeout=5) as resp: + data = json.loads(resp.read().decode()) + tags = [t['name'] for t in data.get('results', []) if not t['name'].endswith('-debug') and not 'beta' in t['name']] + return tags[:8] + except Exception as e: + return [f"Err: {e}"] + +print("Grafana tags:", get_dockerhub_latest("grafana/grafana")) +print("Uptime Kuma tags:", get_dockerhub_latest("louislam/uptime-kuma")) +print("Semaphore tags:", get_dockerhub_latest("semaphoreui/semaphore")) + diff --git a/scripts/check_upstream_tags2.py b/scripts/check_upstream_tags2.py new file mode 100644 index 0000000..c5d092a --- /dev/null +++ b/scripts/check_upstream_tags2.py @@ -0,0 +1,16 @@ +import urllib.request +import json + +def get_tags(repo): + url = f"https://hub.docker.com/v2/repositories/{repo}/tags?page_size=15" + try: + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, timeout=5) as resp: + data = json.loads(resp.read().decode()) + tags = [t['name'] for t in data.get('results', []) if not t['name'].endswith('-debug') and not 'beta' in t['name']] + return tags[:8] + except Exception as e: + return [f"Err: {e}"] + +print("Jenkins tags:", get_tags("jenkins/jenkins")) + diff --git a/scripts/check_wrk03_fail.py b/scripts/check_wrk03_fail.py new file mode 100644 index 0000000..2109ee7 --- /dev/null +++ b/scripts/check_wrk03_fail.py @@ -0,0 +1,27 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=134) +events = job.job_events.filter(event='runner_on_failed') +for e in events: + print(f"FAILED Host: {e.event_data.get('host')}, Task: {e.event_data.get('task')}") + print(f"Res: {e.event_data.get('res')}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) + +_, out, _ = ssh.exec_command('kubectl get pods -A --field-selector spec.nodeName=sninfr2lnxwrk03') +print("Pods on wrk03:\n", out.read().decode()) + +ssh.close() + diff --git a/scripts/check_zabbix_status.py b/scripts/check_zabbix_status.py new file mode 100644 index 0000000..9a1d0f7 --- /dev/null +++ b/scripts/check_zabbix_status.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command('kubectl get pods -n monitoring -o wide') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/check_zabbix_tags.py b/scripts/check_zabbix_tags.py new file mode 100644 index 0000000..74a4b3f --- /dev/null +++ b/scripts/check_zabbix_tags.py @@ -0,0 +1,19 @@ +import urllib.request +import json + +tags_to_check = [ + "zabbix-server-pgsql", + "zabbix-agent2", + "zabbix-web-nginx-pgsql", + "zabbix-web-service" +] + +for repo in tags_to_check: + url = f"https://registry.hub.docker.com/v2/repositories/zabbix/{repo}/tags/ubuntu-7.0.32" + try: + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req) as resp: + print(f"[OK] zabbix/{repo}:ubuntu-7.0.32 exists") + except Exception as e: + print(f"[ERR] zabbix/{repo}:ubuntu-7.0.32 -> {e}") + diff --git a/scripts/cleanup_unknown_pods.py b/scripts/cleanup_unknown_pods.py new file mode 100644 index 0000000..95d2f8c --- /dev/null +++ b/scripts/cleanup_unknown_pods.py @@ -0,0 +1,24 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +for pod in ['pihole-85b7bd696d-ltrvg', 'technitium-dns-6dcf8fc9f6-mxd7p']: + cmd = f'kubectl patch pod -n infra {pod} -p "{{\\"metadata\\":{{\\"finalizers\\":null}}}}" --type=merge' + stdin, stdout, stderr = ssh.exec_command(cmd) + print(f"Patch {pod}:", stdout.read().decode().strip(), stderr.read().decode().strip()) + + del_cmd = f'kubectl delete pod -n infra {pod} --force --grace-period=0' + stdin, stdout, stderr = ssh.exec_command(del_cmd) + print(f"Delete {pod}:", stdout.read().decode().strip(), stderr.read().decode().strip()) + +# Check remaining pods in infra +_, out, _ = ssh.exec_command('kubectl get pods -n infra') +print("\nInfra pods:\n", out.read().decode()) + +ssh.close() + diff --git a/scripts/comprehensive_health_sweep.py b/scripts/comprehensive_health_sweep.py new file mode 100644 index 0000000..8d9d4d1 --- /dev/null +++ b/scripts/comprehensive_health_sweep.py @@ -0,0 +1,45 @@ +import urllib.request +import ssl +import sys + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +endpoints = [ + ("Rancher Console", "https://aerodrome.snarfnet.net"), + ("Argo CD Web UI", "https://argo.snarfnet.net"), + ("Portainer Server", "https://140.44.4.71:9443"), + ("Authentik SSO Portal", "https://authentik.snarfnet.net"), + ("AWX Controller", "https://awx.snarfnet.net"), + ("Grafana Dashboards", "https://grafana.snarfnet.net"), + ("Homepage Dashboard", "https://homepage.snarfnet.net"), + ("Uptime Kuma Monitor", "https://uptime.snarfnet.net/dashboard"), + ("Jenkins Controller", "https://jenkins.snarfnet.net/login"), + ("Coder Workspace", "https://coder.snarfnet.net"), + ("Gitea GitOps Service", "http://140.44.4.51:90"), + ("Zabbix Web Console", "http://140.44.4.51:8888"), + ("Heimdall Portal", "http://140.44.4.71:85"), + ("Peanut Dashboard", "http://140.44.4.71:8050"), + ("Homebox Inventory", "http://140.44.4.71:3100"), + ("Semaphore UI", "https://semaphore.snarfnet.net"), + ("NPM Proxy Admin", "http://140.44.4.71:81") +] + +print(f"{'SERVICE':<25} {'URL':<40} {'STATUS':<15}") +print("=" * 80) + +for name, url in endpoints: + try: + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, timeout=8, context=ctx) as resp: + code = resp.getcode() + print(f"{name:<25} {url:<40} {code} OK") + except urllib.error.HTTPError as e: + # Some endpoints return 401 or 403 on root if auth required + if e.code in [200, 301, 302, 307, 401, 403]: + print(f"{name:<25} {url:<40} {e.code} Responding") + else: + print(f"{name:<25} {url:<40} {e.code} HTTP Error") + except Exception as e: + print(f"{name:<25} {url:<40} ERR: {e}") diff --git a/scripts/deep_search_coder.py b/scripts/deep_search_coder.py new file mode 100644 index 0000000..24f667e --- /dev/null +++ b/scripts/deep_search_coder.py @@ -0,0 +1,20 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.52', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +print("Searching for Coder postgres data on wrk02 (140.44.4.52)...") +cmd = """find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -name "coderv2" 2>/dev/null""" +_, out, err = ssh.exec_command(cmd) +res = out.read().decode().strip() +if res: + print("Found coderv2 paths:\n", res) +else: + print("No coderv2 found directly. Searching for pg_control...") + cmd2 = """find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path "*coderv2*" 2>/dev/null | head -n 20""" + _, out2, _ = ssh.exec_command(cmd2) + print("Search result:\n", out2.read().decode().strip()) + +ssh.close() + diff --git a/scripts/diagnose_argo.py b/scripts/diagnose_argo.py new file mode 100644 index 0000000..3bdc4a3 --- /dev/null +++ b/scripts/diagnose_argo.py @@ -0,0 +1,68 @@ +import socket +import paramiko + +def check_dns(): + print("--- DNS Check ---") + try: + ip = socket.gethostbyname("argo.snarfnet.net") + print(f"argo.snarfnet.net resolves to: {ip}") + except Exception as e: + print(f"DNS lookup error: {e}") + +def check_npm(): + print("\n--- NPM Database Check (140.44.4.71) ---") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + # Check sqlite db or active nginx config in npm + cmd = """sudo python3 -c "import sqlite3; con = sqlite3.connect('/mnt/nfs_shares/nginx_proxy_manager/database.sqlite'); cur = con.cursor(); print(list(cur.execute(\\"SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled, is_deleted FROM proxy_host WHERE domain_names LIKE '%argo%'\\")))" """ + stdin, stdout, stderr = ssh.exec_command(cmd) + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + print("NPM DB Proxy Host for argo:") + print(out) + if err: + print("Error/stderr:", err) + + # Check generated nginx config files + cmd2 = "sudo grep -rn 'argo.snarfnet.net' /mnt/nfs_shares/nginx_proxy_manager/nginx/proxy_host/ || true" + stdin, stdout, stderr = ssh.exec_command(cmd2) + print("NPM config match:") + print(stdout.read().decode().strip()) + + ssh.close() + +def check_spaceport_argo(): + print("\n--- Spaceport ArgoCD K8s Check (140.44.4.41) ---") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + cmd = "kubectl get ingress,svc,pod -n infrastructure | grep -i argo" + stdin, stdout, stderr = ssh.exec_command(cmd) + print("Spaceport Argo Resources:") + print(stdout.read().decode().strip()) + + # Check Ingress YAML + cmd2 = "kubectl get ingress -n infrastructure -o yaml" + stdin, stdout, stderr = ssh.exec_command(cmd2) + ingress_yaml = stdout.read().decode().strip() + print("\nAll Ingresses in infrastructure:") + for line in ingress_yaml.splitlines(): + if "name:" in line or "host:" in line or "service:" in line or "port:" in line: + print(" ", line) + + ssh.close() + +if __name__ == '__main__': + check_dns() + try: + check_npm() + except Exception as e: + print("NPM check failed:", e) + try: + check_spaceport_argo() + except Exception as e: + print("Spaceport check failed:", e) + diff --git a/scripts/diagnose_issues.py b/scripts/diagnose_issues.py new file mode 100644 index 0000000..bbeb3fa --- /dev/null +++ b/scripts/diagnose_issues.py @@ -0,0 +1,52 @@ +import paramiko + +def check_spaceport(): + print("================ SPACEPORT (Coder & Semaphore) ================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + def run(cmd): + print(f"\n--- {cmd} ---") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + + run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide") + run("kubectl logs -n infrastructure -l app.kubernetes.io/name=coder --tail=40") + run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o wide") + run("kubectl logs -n infrastructure -l app.kubernetes.io/name=semaphore --tail=40") + run("curl -k -I https://coder.snarfnet.net") + run("curl -k -I https://semaphore.snarfnet.net") + ssh.close() + +def check_swarm(): + print("\n================ SWARM (Peanut & NPM) ================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + def run(cmd): + print(f"\n--- {cmd} ---") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + + run("sudo docker service ps peanut_peanut --no-trunc") + run("sudo docker service logs peanut_peanut --tail=40") + run("curl -s -I http://140.44.4.70:8050 || curl -s -I http://140.44.4.71:8050") + run("curl -k -I https://peanut.snarfnet.net || true") + run("sudo sqlite3 /mnt/nas_docker/npm/data/database.sqlite 'select id,domain_names,forward_host,forward_port,enabled from proxy_host where domain_names like \"%peanut%\" or domain_names like \"%coder%\" or domain_names like \"%semaphore%\";'") + ssh.close() + +check_spaceport() +check_swarm() + diff --git a/scripts/diagnose_wrk07.py b/scripts/diagnose_wrk07.py new file mode 100644 index 0000000..b231cbb --- /dev/null +++ b/scripts/diagnose_wrk07.py @@ -0,0 +1,23 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("=== NODES ===") +_, out, _ = ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +print("=== PODS ON SNINFR2LNXWRK07 ===") +_, out, _ = ssh.exec_command('kubectl get pods -A -o wide | grep sninfr2lnxwrk07') +print(out.read().decode()) + +print("=== ALL PODS NOT RUNNING / NOT COMPLETED ===") +_, out, _ = ssh.exec_command('kubectl get pods -A | grep -v Running | grep -v Completed') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/export_gitops_manifests.py b/scripts/export_gitops_manifests.py new file mode 100644 index 0000000..434e28b --- /dev/null +++ b/scripts/export_gitops_manifests.py @@ -0,0 +1,93 @@ +import paramiko +import yaml +import os + +CLEAN_METADATA_KEYS = {'uid', 'resourceVersion', 'generation', 'creationTimestamp', 'managedFields', 'selfLink'} +CLEAN_ANNOTATIONS_PREFIXES = { + 'kubectl.kubernetes.io/last-applied-configuration', + 'deployment.kubernetes.io/revision', + 'field.cattle.io/publicEndpoints' +} + +def clean_doc(doc): + if not isinstance(doc, dict): + return doc + doc.pop('status', None) + meta = doc.get('metadata', {}) + for k in CLEAN_METADATA_KEYS: + meta.pop(k, None) + ann = meta.get('annotations', {}) + keys_to_del = [k for k in ann if any(k.startswith(p) for p in CLEAN_ANNOTATIONS_PREFIXES)] + for k in keys_to_del: + ann.pop(k, None) + if not ann: + meta.pop('annotations', None) + return doc + +def dump_resources(ssh, query_cmd, out_file): + print(f"Querying: {query_cmd}") + _, out, err = ssh.exec_command(query_cmd) + raw = out.read().decode('utf-8', errors='replace') + if not raw.strip(): + print(f" Warning: No resources found for {query_cmd}") + return + + docs = yaml.safe_load_all(raw) + cleaned = [] + for d in docs: + if not d: continue + if d.get('kind') == 'List': + for item in d.get('items', []): + cleaned.append(clean_doc(item)) + else: + cleaned.append(clean_doc(d)) + + if cleaned: + os.makedirs(os.path.dirname(out_file), exist_ok=True) + with open(out_file, 'w', encoding='utf-8') as fp: + yaml.dump_all(cleaned, fp, default_flow_style=False, sort_keys=False) + print(f" Saved {len(cleaned)} resources -> {out_file}") + +# 1. Connect to Spaceport +print("\n--- Connecting to Spaceport (140.44.4.41) ---") +ssh_sp = paramiko.SSHClient() +ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +base_sp = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport" + +# Homepage +dump_resources(ssh_sp, "kubectl get deploy,svc,ing,cm -n infrastructure -l app.kubernetes.io/name=homepage-2-1744068158 -o yaml", os.path.join(base_sp, "homepage", "homepage.yaml")) +# Jenkins +dump_resources(ssh_sp, "kubectl get sts,svc,ing -n infrastructure -l app.kubernetes.io/name=jenkins -o yaml", os.path.join(base_sp, "jenkins", "jenkins.yaml")) +# Semaphore +dump_resources(ssh_sp, "kubectl get deploy,svc,ing,cm -n infrastructure -l app.kubernetes.io/name=semaphore -o yaml", os.path.join(base_sp, "semaphore", "semaphore.yaml")) +# Coder +dump_resources(ssh_sp, "kubectl get deploy,svc,ing -n infrastructure -l app.kubernetes.io/name=coder -o yaml", os.path.join(base_sp, "coder", "coder.yaml")) +# Gitea +dump_resources(ssh_sp, "kubectl get deploy,sts,svc,ing,cm -n infrastructure -l app.kubernetes.io/instance=gitea -o yaml", os.path.join(base_sp, "gitea", "gitea.yaml")) +# Zabbix +dump_resources(ssh_sp, "kubectl get deploy,sts,svc,ing -n monitoring -l app.kubernetes.io/instance=zabbix -o yaml", os.path.join(base_sp, "zabbix", "zabbix.yaml")) + +ssh_sp.close() + +# 2. Connect to Aerodrome +print("\n--- Connecting to Aerodrome (140.44.4.11) ---") +ssh_ae = paramiko.SSHClient() +ssh_ae.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_ae.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +base_ae = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome" + +# Grafana +dump_resources(ssh_ae, "kubectl get deploy,svc,ing -n default -l app.kubernetes.io/name=grafana -o yaml", os.path.join(base_ae, "grafana", "grafana.yaml")) +# Uptime Kuma +dump_resources(ssh_ae, "kubectl get deploy,svc,ing -n monitoring -l app.kubernetes.io/name=uptime-kuma -o yaml", os.path.join(base_ae, "uptime-kuma", "uptime-kuma.yaml")) +# AWX +dump_resources(ssh_ae, "kubectl get awx,deploy,svc,ing -n awx -o yaml", os.path.join(base_ae, "awx", "awx.yaml")) +# Authentik +dump_resources(ssh_ae, "kubectl get deploy,sts,svc,ing -n infra -l app.kubernetes.io/instance=authentik -o yaml", os.path.join(base_ae, "authentik", "authentik.yaml")) + +ssh_ae.close() + +print("\nManifest extraction complete!") diff --git a/scripts/find_coder_data.py b/scripts/find_coder_data.py new file mode 100644 index 0000000..7d9e831 --- /dev/null +++ b/scripts/find_coder_data.py @@ -0,0 +1,20 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.52', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +run("crictl ps -a") +run("find /var/lib/rancher/k3s/agent/containerd/ -type d -name 'coderv2' 2>/dev/null") +run("find /run/k3s/containerd/ -type d -name 'coderv2' 2>/dev/null") +run("find / -name 'database.boltdb' 2>/dev/null") + +ssh.close() + diff --git a/scripts/fix_and_upgrade_uptime.py b/scripts/fix_and_upgrade_uptime.py new file mode 100644 index 0000000..712242b --- /dev/null +++ b/scripts/fix_and_upgrade_uptime.py @@ -0,0 +1,36 @@ +import paramiko +import time + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +# 1. Update strategy to Recreate +run("""kubectl patch deployment uptime-kuma-2-1756668961 -n monitoring --type='json' -p='[{"op": "replace", "path": "/spec/strategy", "value": {"type": "Recreate"}}]'""") + +# 2. Scale down to 0 to release all locks cleanly +print("[*] Scaling down to 0 to release all volume locks...") +run("kubectl scale deployment uptime-kuma-2-1756668961 -n monitoring --replicas=0") +run("kubectl wait --for=delete pod -l app.kubernetes.io/name=uptime-kuma -n monitoring --timeout=60s || true") + +# 3. Set image to 2.5.9 +run("kubectl set image deployment/uptime-kuma-2-1756668961 uptime-kuma=louislam/uptime-kuma:2.5.9 -n monitoring") + +# 4. Scale up to 1 +print("[*] Scaling up to 1...") +run("kubectl scale deployment uptime-kuma-2-1756668961 -n monitoring --replicas=1") +run("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=120s") +run("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma -o wide") + +ssh.close() + diff --git a/scripts/fix_argo_npm.py b/scripts/fix_argo_npm.py new file mode 100644 index 0000000..4823c8e --- /dev/null +++ b/scripts/fix_argo_npm.py @@ -0,0 +1,58 @@ +import paramiko + +def fix_argo(): + # 1. Update on swm01 (140.44.4.71) + ssh1 = paramiko.SSHClient() + ssh1.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh1.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + print("--- Backing up SQLite DB and 39.conf ---") + cmd_backup = """ + sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c " + cp /data/database.sqlite /data/database.sqlite.bak.argo && + cp /data/nginx/proxy_host/39.conf /data/nginx/proxy_host/39.conf.bak + " + """ + stdin, stdout, stderr = ssh1.exec_command(cmd_backup) + print("Backup stdout:", stdout.read().decode()) + print("Backup stderr:", stderr.read().decode()) + + print("--- Updating SQLite DB ---") + cmd_db = """ + sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c " + python3 -c \\\"import sqlite3; con = sqlite3.connect('/data/database.sqlite'); cur = con.cursor(); cur.execute('UPDATE proxy_host SET forward_host=\\\'140.44.4.62\\\' WHERE id=39'); con.commit(); con.close(); print('DB updated successfully')\\\" + " + """ + stdin, stdout, stderr = ssh1.exec_command(cmd_db) + print("DB stdout:", stdout.read().decode()) + print("DB stderr:", stderr.read().decode()) + + print("--- Updating 39.conf ---") + cmd_conf = """ + sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') sh -c " + sed -i 's/140.44.4.51/140.44.4.62/g' /data/nginx/proxy_host/39.conf + " + """ + stdin, stdout, stderr = ssh1.exec_command(cmd_conf) + print("Conf sed stdout:", stdout.read().decode()) + print("Conf sed stderr:", stderr.read().decode()) + + print("--- Reloading NGINX on swm01 ---") + stdin, stdout, stderr = ssh1.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload") + print("Reload swm01 stdout:", stdout.read().decode()) + print("Reload swm01 stderr:", stderr.read().decode()) + ssh1.close() + + # 2. Reload on swm02 (140.44.4.72) + ssh2 = paramiko.SSHClient() + ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh2.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + print("--- Reloading NGINX on swm02 ---") + stdin, stdout, stderr = ssh2.exec_command("sudo docker exec $(sudo docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload") + print("Reload swm02 stdout:", stdout.read().decode()) + print("Reload swm02 stderr:", stderr.read().decode()) + ssh2.close() + +if __name__ == '__main__': + fix_argo() + diff --git a/scripts/fix_cluster_issues.py b/scripts/fix_cluster_issues.py new file mode 100644 index 0000000..d4aa09e --- /dev/null +++ b/scripts/fix_cluster_issues.py @@ -0,0 +1,34 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +# 1. Install open-iscsi on sninfr2lnxwrk01 (140.44.4.21) +print("=== INSTALLING OPEN-ISCSI ON WRK01 (140.44.4.21) ===") +ssh_wrk01 = paramiko.SSHClient() +ssh_wrk01.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_wrk01.connect('140.44.4.21', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) +stdin, stdout, stderr = ssh_wrk01.exec_command('apt-get update -qq && apt-get install -y open-iscsi nfs-common && systemctl enable --now iscsid') +print(stdout.read().decode()) +print("iscsid status:", end=" ") +_, out, _ = ssh_wrk01.exec_command('systemctl is-active iscsid') +print(out.read().decode().strip()) +ssh_wrk01.close() + +# 2. Check on Master +ssh_master = paramiko.SSHClient() +ssh_master.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_master.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("=== DELETING CRASHING LONGHORN-MANAGER-SQTL9 POD SO IT RECREATES WITH ISCSID ===") +ssh_master.exec_command('kubectl delete pod -n longhorn-system longhorn-manager-sqtl9') + +print("=== CHECKING HELM-INSTALL-TRAEFIK-CRD LOGS ===") +_, out, _ = ssh_master.exec_command('kubectl logs -n kube-system helm-install-traefik-crd-jt9zk --tail=25') +print(out.read().decode()) + +print("=== CLEANING 130-DAY UNKNOWN PODS IN INFRA ===") +ssh_master.exec_command('kubectl delete pod -n infra pihole-85b7bd696d-ltrvg technitium-dns-6dcf8fc9f6-mxd7p --force --grace-period=0') + +ssh_master.close() + diff --git a/scripts/fix_traefik_and_verify.py b/scripts/fix_traefik_and_verify.py new file mode 100644 index 0000000..1b78f9f --- /dev/null +++ b/scripts/fix_traefik_and_verify.py @@ -0,0 +1,41 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("=== FIXING TRAEFIK CRD LABELS & ANNOTATIONS ===") +annotate_cmd = """ +for crd in $(kubectl get crd -o name | grep 'traefik.io'); do + echo "Patching $crd..." + kubectl annotate $crd meta.helm.sh/release-name=traefik-crd meta.helm.sh/release-namespace=kube-system --overwrite + kubectl label $crd app.kubernetes.io/managed-by=Helm --overwrite +done +""" +stdin, stdout, stderr = ssh.exec_command(annotate_cmd) +print(stdout.read().decode()) + +print("=== DELETING FAILED HELM INSTALL JOBS TO RETRIGGER ===") +ssh.exec_command('kubectl delete job -n kube-system helm-install-traefik-crd helm-install-traefik') + +print("Waiting 15 seconds for Helm controller to reconcile...") +time.sleep(15) + +print("=== CHECKING LONGHORN-MANAGER ON WRK01 ===") +_, out, _ = ssh.exec_command('kubectl get pods -n longhorn-system -l app=longhorn-manager -o wide') +print(out.read().decode()) + +print("=== CHECKING ALL PODS IN KUBE-SYSTEM ===") +_, out, _ = ssh.exec_command('kubectl get pods -n kube-system | grep traefik') +print(out.read().decode()) + +print("=== CHECKING ANY REMAINING CRASHING PODS ===") +_, out, _ = ssh.exec_command('kubectl get pods -A | grep -v Running | grep -v Completed') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/fix_uptime_kuma.py b/scripts/fix_uptime_kuma.py new file mode 100644 index 0000000..b187f99 --- /dev/null +++ b/scripts/fix_uptime_kuma.py @@ -0,0 +1,18 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + print(out.read().decode().strip()) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +run("kubectl get deployment uptime-kuma-2-1756668961 -n monitoring -o yaml | grep -A 4 strategy:") +run("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma") +ssh.close() + diff --git a/scripts/get_coder_env.py b/scripts/get_coder_env.py new file mode 100644 index 0000000..24f28ae --- /dev/null +++ b/scripts/get_coder_env.py @@ -0,0 +1,21 @@ +import paramiko +import json + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, err = ssh.exec_command('kubectl get pod -n infrastructure -l app.kubernetes.io/name=coder -o json') +pod = json.loads(out.read().decode()) +if pod.get('items'): + container = pod['items'][0]['spec']['containers'][0] + print("Image:", container.get('image')) + print("Env:") + for e in container.get('env', []): + print(f" {e.get('name')}: {e.get('value', e.get('valueFrom'))}") + print("VolumeMounts:") + for vm in container.get('volumeMounts', []): + print(f" {vm.get('name')} -> {vm.get('mountPath')}") + +ssh.close() + diff --git a/scripts/inspect_argocd_apps.py b/scripts/inspect_argocd_apps.py new file mode 100644 index 0000000..674749b --- /dev/null +++ b/scripts/inspect_argocd_apps.py @@ -0,0 +1,15 @@ +import glob +import yaml + +files = glob.glob(r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\argocd\**\*.yaml", recursive=True) +for f in files: + with open(f, 'r') as fp: + docs = yaml.safe_load_all(fp) + for doc in docs: + if not doc: continue + if doc.get('kind') == 'Application': + name = doc.get('metadata', {}).get('name') + dest = doc.get('spec', {}).get('destination', {}) + src = doc.get('spec', {}).get('source', {}) + print(f"App: {name:<30} Path: {src.get('path'):<35} Server: {dest.get('server'):<35} NS: {dest.get('namespace')}") + diff --git a/scripts/inspect_authentik.py b/scripts/inspect_authentik.py new file mode 100644 index 0000000..6b50813 --- /dev/null +++ b/scripts/inspect_authentik.py @@ -0,0 +1,18 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + +run("kubectl get deploy,sts,pod -n infra -o wide") +run("kubectl get deploy authentik-server -n infra -o yaml | grep -E 'image:|strategy:' -A 5") + +ssh.close() + diff --git a/scripts/inspect_coder_data_to_persist.py b/scripts/inspect_coder_data_to_persist.py new file mode 100644 index 0000000..79afb5a --- /dev/null +++ b/scripts/inspect_coder_data_to_persist.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder") +run("kubectl exec -n infrastructure deployment/coder -- ls -la /home/coder") +run("kubectl exec -n infrastructure deployment/coder -- du -sh /home/coder/.config/coderv2") +run("curl -s https://coder.snarfnet.net/api/v2/users/first") + +ssh.close() + diff --git a/scripts/inspect_coder_helm.py b/scripts/inspect_coder_helm.py new file mode 100644 index 0000000..7218273 --- /dev/null +++ b/scripts/inspect_coder_helm.py @@ -0,0 +1,21 @@ +import paramiko +import base64 +import gzip +import json + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command('kubectl get secret -n infrastructure sh.helm.release.v1.coder.v5 -o jsonpath="{.data.release}"') +raw = out.read().decode().strip() +if raw: + data = base64.b64decode(raw) + data = base64.b64decode(data) # helm encodes twice + decomp = gzip.decompress(data) + rel = json.loads(decomp) + print("Chart:", rel.get('chart', {}).get('metadata', {}).get('version')) + print("Values:", json.dumps(rel.get('config', {}), indent=2)) + +ssh.close() + diff --git a/scripts/inspect_frontend_fd.py b/scripts/inspect_frontend_fd.py new file mode 100644 index 0000000..7300fe8 --- /dev/null +++ b/scripts/inspect_frontend_fd.py @@ -0,0 +1,19 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('ls -l /proc/3712358/fd') +print("--- FD OF LINUX-CHECK-REMOVAL ---") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('ls -l /proc/3712345/fd') +print("--- FD OF FRONTEND ---") +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_gotify_db.py b/scripts/inspect_gotify_db.py new file mode 100644 index 0000000..7bbdda8 --- /dev/null +++ b/scripts/inspect_gotify_db.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """sudo python3 -c ' +import sqlite3 +conn = sqlite3.connect("/mnt/swarm_shared/legacy_volumes/gotify/gotify.db") +cur = conn.cursor() +print("Applications:", cur.execute("SELECT id, name, description FROM applications").fetchall()) +print("Clients:", cur.execute("SELECT id, name FROM clients").fetchall()) +print("Messages count:", cur.execute("SELECT count(*) FROM messages").fetchone()[0]) +print("Recent messages:", cur.execute("SELECT id, title, message, date FROM messages ORDER BY id DESC LIMIT 5").fetchall()) +print("Users:", cur.execute("SELECT id, name, admin FROM users").fetchall()) +'""" + +_, out, err = ssh.exec_command(py_script) +print(out.read().decode()) +print("ERR:", err.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_helm_and_images.py b/scripts/inspect_helm_and_images.py new file mode 100644 index 0000000..9c1b2a6 --- /dev/null +++ b/scripts/inspect_helm_and_images.py @@ -0,0 +1,28 @@ +import paramiko + +ssh_sp = paramiko.SSHClient() +ssh_sp.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_sp.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +ssh_ae = paramiko.SSHClient() +ssh_ae.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh_ae.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(ssh, name, cmd): + print(f"\n--- {name}: {cmd} ---") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + +run(ssh_sp, "SPACEPORT", "helm list -A") +run(ssh_sp, "SPACEPORT", "kubectl get deploy -n monitoring -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'") + +run(ssh_ae, "AERODROME", "helm list -A") +run(ssh_ae, "AERODROME", "kubectl get deploy -n infra -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'") +run(ssh_ae, "AERODROME", "kubectl get deploy -n awx -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'") +run(ssh_ae, "AERODROME", "kubectl get awx -n awx -o yaml | grep -E 'image|version'") + +ssh_sp.close() +ssh_ae.close() + diff --git a/scripts/inspect_homepage_k8s.py b/scripts/inspect_homepage_k8s.py new file mode 100644 index 0000000..3719b51 --- /dev/null +++ b/scripts/inspect_homepage_k8s.py @@ -0,0 +1,11 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command('kubectl get all,ing,cm,pvc -n infrastructure | grep -i homepage') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_job_events.py b/scripts/inspect_job_events.py new file mode 100644 index 0000000..f1463f6 --- /dev/null +++ b/scripts/inspect_job_events.py @@ -0,0 +1,24 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_check = """ +from awx.main.models import Job +job = Job.objects.get(id=131) +print(f"Result Traceback: {job.result_traceback}") +events = job.job_events.filter(event__in=['runner_on_ok', 'runner_on_failed', 'playbook_on_stats']).order_by('-id')[:20] +for e in events: + print(f"[{e.event}] Host: {e.event_data.get('host')} - Task: {e.event_data.get('task')}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_check}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_jt10.py b/scripts/inspect_jt10.py new file mode 100644 index 0000000..a4eb1a8 --- /dev/null +++ b/scripts/inspect_jt10.py @@ -0,0 +1,23 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +py_code = """ +from awx.main.models import JobTemplate +jt = JobTemplate.objects.get(id=10) +print(f"Name: {jt.name}") +print(f"Playbook: {jt.playbook}") +print(f"Inventory: {jt.inventory.name if jt.inventory else None}") +print(f"Credential: {[c.name for c in jt.credentials.all()]}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/inspect_npm_hosts.py b/scripts/inspect_npm_hosts.py new file mode 100644 index 0000000..55abc8d --- /dev/null +++ b/scripts/inspect_npm_hosts.py @@ -0,0 +1,18 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """sudo python3 -c ' +import sqlite3 +conn = sqlite3.connect("/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite") +cur = conn.cursor() +for row in cur.execute("select id,domain_names,forward_host,forward_port,enabled from proxy_host"): + print(row) +'""" +_, out, err = ssh.exec_command(py_script) +print(out.read().decode()) +print("ERR:", err.read().decode()) + +ssh.close() diff --git a/scripts/inspect_peanut_simple.py b/scripts/inspect_peanut_simple.py new file mode 100644 index 0000000..a6c59a7 --- /dev/null +++ b/scripts/inspect_peanut_simple.py @@ -0,0 +1,17 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +_, out, _ = ssh.exec_command("sudo docker service inspect peanut_peanut --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'") +print("Mounts:", out.read().decode()) + +_, out2, _ = ssh.exec_command("ls -la /mnt/swarm_shared/legacy_volumes/peanut/") +print("Files:", out2.read().decode()) + +_, out3, _ = ssh.exec_command("cat /mnt/swarm_shared/legacy_volumes/peanut/settings.yml") +print("Settings:", out3.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_peanut_stack.py b/scripts/inspect_peanut_stack.py new file mode 100644 index 0000000..df1173b --- /dev/null +++ b/scripts/inspect_peanut_stack.py @@ -0,0 +1,19 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + +run("sudo docker service inspect peanut_peanut --format '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}'") +run("find /mnt/ -name 'settings.yml' 2>/dev/null") +run("find /mnt/ -name '*peanut*' 2>/dev/null") + +ssh.close() + diff --git a/scripts/inspect_portainer.py b/scripts/inspect_portainer.py new file mode 100644 index 0000000..a902b56 --- /dev/null +++ b/scripts/inspect_portainer.py @@ -0,0 +1,29 @@ +import paramiko +import sys +import json + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('sudo docker inspect portainer') +data = json.loads(stdout.read().decode())[0] + +print("Container Name:", data["Name"]) +print("Image:", data["Config"]["Image"]) +print("Command:", data["Config"]["Cmd"]) +print("Mounts:") +for m in data.get("Mounts", []): + print(" ", m.get("Source"), "->", m.get("Destination")) + +stdin, stdout, stderr = ssh.exec_command('sudo docker exec portainer /portainer --version || sudo docker inspect --format="{{.Config.Labels}}" portainer') +print("Version/Labels:", stdout.read().decode().strip()) + +# Check portainer_agent service +stdin, stdout, stderr = ssh.exec_command('sudo docker service inspect portainer_agent --format="{{.Spec.TaskTemplate.ContainerSpec.Image}}"') +print("Agent Service Image:", stdout.read().decode().strip()) + +ssh.close() + diff --git a/scripts/inspect_prerm.py b/scripts/inspect_prerm.py new file mode 100644 index 0000000..e8d11fa --- /dev/null +++ b/scripts/inspect_prerm.py @@ -0,0 +1,23 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('pstree -p 3700051 || ps -ef | grep 3712344') +print("--- PROCESS TREE ---") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('cat /proc/3712344/cmdline; echo ""; cat /proc/3712344/wchan') +print("--- WCHAN ---") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('cat /var/lib/dpkg/info/linux-image-6.6.62+rpt-rpi-v8.prerm') +print("--- PRERM SCRIPT ---") +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_rancher.py b/scripts/inspect_rancher.py new file mode 100644 index 0000000..ce01c92 --- /dev/null +++ b/scripts/inspect_rancher.py @@ -0,0 +1,20 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('kubectl get deployment -n cattle-system rancher -o jsonpath="{.spec.template.spec.containers[0].image}"') +print("Rancher image:", stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('kubectl get clusters.management.cattle.io -o custom-columns=NAME:.metadata.name,DISPLAY:.spec.displayName,K8S_VER:.status.version.gitVersion') +print("\nRancher Managed Clusters:\n", stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('kubectl get nodes') +print("Direct kubectl nodes on Aerodrome:\n", stdout.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_semaphore_boltdb.py b/scripts/inspect_semaphore_boltdb.py new file mode 100644 index 0000000..147c394 --- /dev/null +++ b/scripts/inspect_semaphore_boltdb.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_script = """python3 -c ' +# Simple BoltDB reader without external dependencies +# Or let us inspect with strings or hexdump +import os +print("BoltDB file size:", os.path.getsize("/var/lib/semaphore/database.boltdb")) +'""" + +# In the pod, let us see if boltdb has project records +cmd = """kubectl exec -n infrastructure deployment/semaphore -c semaphore -- /bin/sh -c ' +strings /var/lib/semaphore/database.boltdb | grep -E "name|title|playbook|project" | head -n 40 +'""" +_, out, err = ssh.exec_command(cmd) +print("Strings from Semaphore BoltDB:") +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_spaceport_workloads.py b/scripts/inspect_spaceport_workloads.py new file mode 100644 index 0000000..f4b4804 --- /dev/null +++ b/scripts/inspect_spaceport_workloads.py @@ -0,0 +1,28 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +# Check Coder +run("kubectl get deploy,sts,pod -n infrastructure -l app.kubernetes.io/name=coder -o wide") +run("kubectl get deploy,sts,pod -n infrastructure | grep -E 'coder|gitea|zabbix'") + +# Inspect Gitea details +run("kubectl get deploy,sts -n infrastructure -o wide") + +# Check Coder images +run("kubectl get deploy -n infrastructure -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .spec.template.spec.containers[*]}{.image}{\" \"}{end}{\"\\n\"}'") + +ssh.close() + diff --git a/scripts/inspect_swarm_digests.py b/scripts/inspect_swarm_digests.py new file mode 100644 index 0000000..44aa862 --- /dev/null +++ b/scripts/inspect_swarm_digests.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +run("sudo docker service ps heimdall_heimdall --no-trunc") +run("sudo docker service ps gotify_gotify --no-trunc") +run("sudo docker service ps homebox_homebox --no-trunc") +run("sudo docker service ps peanut_peanut --no-trunc") + +ssh.close() + diff --git a/scripts/inspect_swarm_services.py b/scripts/inspect_swarm_services.py new file mode 100644 index 0000000..3b66b56 --- /dev/null +++ b/scripts/inspect_swarm_services.py @@ -0,0 +1,21 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +run("sudo docker service ls") +run("sudo docker stack ls") + +ssh.close() + diff --git a/scripts/inspect_swarm_status.py b/scripts/inspect_swarm_status.py new file mode 100644 index 0000000..1da2395 --- /dev/null +++ b/scripts/inspect_swarm_status.py @@ -0,0 +1,26 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('sudo docker node ls') +print("=== DOCKER NODES ===") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('sudo docker service ls') +print("=== DOCKER SERVICES ===") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('ip addr show | grep 140.44.4') +print("=== IP ADDRESSES ON SWM01 ===") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('systemctl is-active keepalived') +print("Keepalived status:", stdout.read().decode().strip()) + +ssh.close() + diff --git a/scripts/inspect_wrk07_process.py b/scripts/inspect_wrk07_process.py new file mode 100644 index 0000000..7b48034 --- /dev/null +++ b/scripts/inspect_wrk07_process.py @@ -0,0 +1,23 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +stdin, stdout, stderr = ssh.exec_command('ps aux | grep -v "grep" | grep -i apt') +print("--- APT PROCESSES ---") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('ps aux | grep -v "grep" | grep -i dpkg') +print("--- DPKG PROCESSES ---") +print(stdout.read().decode()) + +stdin, stdout, stderr = ssh.exec_command('uptime') +print("--- UPTIME ---") +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/inspect_zabbix_deployments.py b/scripts/inspect_zabbix_deployments.py new file mode 100644 index 0000000..4161abb --- /dev/null +++ b/scripts/inspect_zabbix_deployments.py @@ -0,0 +1,17 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + +run("kubectl get deploy -n monitoring -o jsonpath='{range .items[*]}{\"DEPLOY: \"}{.metadata.name}{\"\\n\"}{range .spec.template.spec.containers[*]}{\" container: \"}{.name}{\" image: \"}{.image}{\"\\n\"}{end}{\"\\n\"}{end}'") + +ssh.close() + diff --git a/scripts/launch_k3s_upgrade.py b/scripts/launch_k3s_upgrade.py new file mode 100644 index 0000000..cb49832 --- /dev/null +++ b/scripts/launch_k3s_upgrade.py @@ -0,0 +1,22 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_launch = """ +from awx.main.models import JobTemplate +jt = JobTemplate.objects.get(id=12) +job = jt.create_unified_job() +job.signal_start() +print(f"Launched Job [{job.id}] for Template '{jt.name}': Status={job.status}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_launch}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/launch_os_patching_job.py b/scripts/launch_os_patching_job.py new file mode 100644 index 0000000..ce3a674 --- /dev/null +++ b/scripts/launch_os_patching_job.py @@ -0,0 +1,24 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +py_launch = """ +from awx.main.models import JobTemplate +jt = JobTemplate.objects.get(id=10) +job = jt.create_unified_job() +job.signal_start() +print(f"Launched Job [{job.id}] for Template '{jt.name}': Status={job.status}") +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_launch}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) + +ssh.close() + diff --git a/scripts/list_awx_templates.py b/scripts/list_awx_templates.py new file mode 100644 index 0000000..53a9d81 --- /dev/null +++ b/scripts/list_awx_templates.py @@ -0,0 +1,20 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +py_code = """ +from awx.main.models import JobTemplate +for jt in JobTemplate.objects.all(): + print(f'ID {jt.id}: {jt.name} (playbook: {jt.playbook})') +""" + +cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF" +stdin, stdout, stderr = ssh.exec_command(cmd) +print(stdout.read().decode()) +ssh.close() + diff --git a/scripts/persist_coder_storage.py b/scripts/persist_coder_storage.py new file mode 100644 index 0000000..b323bdf --- /dev/null +++ b/scripts/persist_coder_storage.py @@ -0,0 +1,131 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"\n=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode('utf-8', errors='replace').strip() + if res: + print(res) + e = err.read().decode('utf-8', errors='replace').strip() + if e: + print("ERR:", e) + +print("[1] Creating backup directory on Spaceport master...") +run("mkdir -p /var/backups/coder_home") + +print("[2] Getting current Coder pod name...") +_, out, _ = ssh.exec_command("kubectl get pod -n infrastructure -l app.kubernetes.io/name=coder -o jsonpath='{.items[0].metadata.name}'") +pod_name = out.read().decode().strip() +print(f"Active Coder pod: {pod_name}") + +print("[3] Copying /home/coder from active pod to /var/backups/coder_home...") +run(f"kubectl exec -n infrastructure {pod_name} -- tar -czf /tmp/coder_backup.tar.gz -C /home/coder .") +run(f"kubectl cp infrastructure/{pod_name}:/tmp/coder_backup.tar.gz /var/backups/coder_home/coder_backup.tar.gz") +run("ls -lh /var/backups/coder_home/coder_backup.tar.gz") + +print("[4] Creating PersistentVolumeClaim 'coder-data' (10Gi, Longhorn)...") +pvc_yaml = """cat </dev/null""" +_, out, err = ssh.exec_command(cmd) +res = out.read().decode().strip() +print("Found on wrk04:\n", res) + +ssh.close() + diff --git a/scripts/test_dns_argo.py b/scripts/test_dns_argo.py new file mode 100644 index 0000000..22b979c --- /dev/null +++ b/scripts/test_dns_argo.py @@ -0,0 +1,21 @@ +import subprocess + +dns_servers = [ + '14.4.4.1', + '140.44.4.65', + '14.4.4.44', + '140.44.4.1', + '140.44.4.70', + '140.44.4.71', + '1.1.1.1', + '8.8.8.8' +] + +for server in dns_servers: + try: + cmd = f"nslookup argo.snarfnet.net {server}" + res = subprocess.run(cmd, capture_output=True, text=True, shell=True, timeout=5) + print(f"=== DNS Server {server} ===") + print(res.stdout.strip()) + except Exception as e: + print(f"Server {server} failed: {e}") diff --git a/scripts/test_swarm_node_patch.py b/scripts/test_swarm_node_patch.py new file mode 100644 index 0000000..3e6484e --- /dev/null +++ b/scripts/test_swarm_node_patch.py @@ -0,0 +1,83 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_IP = '140.44.4.75' +TARGET_NAME = 'snshpd4lnxswm05' + +print(f"=== TESTING ROLLING OS UPDATE ON SWARM NODE {TARGET_NAME} ({TARGET_IP}) ===") + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect(TARGET_IP, username='snarf', key_filename=KEY_FILE, timeout=10) + +upgrade_cmd = """ +echo "debconf debconf/frontend select Noninteractive" | sudo debconf-set-selections +echo "linux-base linux-base/removing-running-kernel boolean false" | sudo debconf-set-selections +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y +sudo apt-get autoremove -y +""" + +print("[*] Running apt dist-upgrade...") +stdin, stdout, stderr = ssh.exec_command(upgrade_cmd) +exit_code = stdout.channel.recv_exit_status() +print(f"[*] apt dist-upgrade exit code: {exit_code}") + +# Check reboot +stdin, stdout, stderr = ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") +reboot_needed = stdout.read().decode().strip() == 'yes' +print(f"[*] Reboot required: {reboot_needed}") + +if reboot_needed: + print("[*] Gracefully rebooting node...") + try: + ssh.exec_command("sudo reboot") + except Exception: + pass + ssh.close() + + print("[*] Waiting 30s for node to reboot...") + time.sleep(30) + + reconnected = False + for attempt in range(25): + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(TARGET_IP, username='snarf', key_filename=KEY_FILE, timeout=6) + reconnected = True + print(f"[*] Node reconnected on attempt {attempt+1}!") + break + except Exception: + time.sleep(4) + + if not reconnected: + print("[!] ERROR: Node failed to reconnect!") + sys.exit(1) + + _, out, _ = ssh.exec_command("sudo systemctl is-active docker") + print(" Docker service:", out.read().decode().strip()) + _, out, _ = ssh.exec_command("systemctl is-active systemd-timesyncd || systemctl is-active chrony") + print(" TimeSync:", out.read().decode().strip()) + _, out, _ = ssh.exec_command("systemctl is-active zabbix-agent2") + print(" ZabbixAgent2:", out.read().decode().strip()) + ssh.close() +else: + ssh.close() + +# Verify on Swarm Leader (140.44.4.73) +leader_ssh = paramiko.SSHClient() +leader_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +leader_ssh.connect('140.44.4.73', username='snarf', key_filename=KEY_FILE, timeout=10) +_, out, _ = leader_ssh.exec_command('sudo docker node ls') +print("\n=== SWARM NODES FROM LEADER ===") +print(out.read().decode()) +leader_ssh.close() + +print(f"[SUCCESS] Swarm node {TARGET_NAME} updated cleanly!") + diff --git a/scripts/uncordon_and_inspect_errors.py b/scripts/uncordon_and_inspect_errors.py new file mode 100644 index 0000000..ef9e14a --- /dev/null +++ b/scripts/uncordon_and_inspect_errors.py @@ -0,0 +1,33 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +print("=== UNCORDONING WRK07 ===") +_, out, _ = ssh.exec_command('kubectl uncordon sninfr2lnxwrk07') +print(out.read().decode()) + +print("=== CHECKING WRK07 STATUS ===") +_, out, _ = ssh.exec_command('kubectl get node sninfr2lnxwrk07') +print(out.read().decode()) + +print("=== INSPECTING LONGHORN-MANAGER-SQTL9 ===") +_, out, _ = ssh.exec_command('kubectl get pod -n longhorn-system longhorn-manager-sqtl9 -o wide') +print(out.read().decode()) +_, out, _ = ssh.exec_command('kubectl logs -n longhorn-system longhorn-manager-sqtl9 -c longhorn-manager --tail=25') +print("Logs:\n", out.read().decode()) + +print("=== INSPECTING HELM-INSTALL-TRAEFIK CRASH ===") +_, out, _ = ssh.exec_command('kubectl logs -n kube-system helm-install-traefik-96s4n --tail=25') +print(out.read().decode()) + +print("=== INSPECTING UNKNOWN PODS IN INFRA ===") +_, out, _ = ssh.exec_command('kubectl get pods -n infra -o wide | grep Unknown') +print(out.read().decode()) + +ssh.close() + diff --git a/scripts/unstick_wrk07.py b/scripts/unstick_wrk07.py new file mode 100644 index 0000000..e8ab050 --- /dev/null +++ b/scripts/unstick_wrk07.py @@ -0,0 +1,27 @@ +import paramiko +import sys + +sys.stdout.reconfigure(encoding='utf-8') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.27', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8) + +unstick_cmd = """ +kill -9 3712358 3712345 3712344 3712307 3700051 2>/dev/null || true +echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections +echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections +export DEBIAN_FRONTEND=noninteractive +dpkg --configure -a +apt-get -f install -y +apt-get autoremove -y +""" +stdin, stdout, stderr = ssh.exec_command(unstick_cmd) +print("STDOUT:\n", stdout.read().decode()) +print("STDERR:\n", stderr.read().decode()) + +stdin, stdout, stderr = ssh.exec_command("ps aux | grep -i apt; ps aux | grep -i dpkg; test -f /var/run/reboot-required && echo 'reboot required' || echo 'no reboot required'") +print("STATUS:\n", stdout.read().decode()) + +ssh.close() + diff --git a/scripts/upgrade_aerodrome_132.py b/scripts/upgrade_aerodrome_132.py new file mode 100644 index 0000000..ab0c5e3 --- /dev/null +++ b/scripts/upgrade_aerodrome_132.py @@ -0,0 +1,141 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.32.13+k3s1' +ENCODED_VER = 'v1.32.13%2Bk3s1' + +masters = [ + ('sninfr2lnxctl01', '140.44.4.11'), + ('sninfr2lnxctl02', '140.44.4.12'), + ('sninfr2lnxctl03', '140.44.4.13') +] + +workers = [ + ('sninfr2lnxwrk01', '140.44.4.21'), + ('sninfr2lnxwrk02', '140.44.4.22'), + ('sninfr2lnxwrk03', '140.44.4.23'), + ('sninfr2lnxwrk04', '140.44.4.24'), + ('sninfr2lnxwrk05', '140.44.4.25'), + ('sninfr2lnxwrk06', '140.44.4.26'), + ('sninfr2lnxwrk07', '140.44.4.27') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING AERODROME UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-132-aerodrome-{int(time.time())}') + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + stdout.channel.recv_exit_status() + + for _ in range(25): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(30): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + stdout.channel.recv_exit_status() + + for _ in range(25): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(30): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL AERODROME NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +print("=== FINAL LONGHORN VOLUMES ===") +_, out, _ = master_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] AERODROME FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_aerodrome_133.py b/scripts/upgrade_aerodrome_133.py new file mode 100644 index 0000000..a5bf7b7 --- /dev/null +++ b/scripts/upgrade_aerodrome_133.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.33.13+k3s2' +ENCODED_VER = 'v1.33.13%2Bk3s2' + +masters = [ + ('sninfr2lnxctl01', '140.44.4.11'), + ('sninfr2lnxctl02', '140.44.4.12'), + ('sninfr2lnxctl03', '140.44.4.13') +] + +workers = [ + ('sninfr2lnxwrk01', '140.44.4.21'), + ('sninfr2lnxwrk02', '140.44.4.22'), + ('sninfr2lnxwrk03', '140.44.4.23'), + ('sninfr2lnxwrk04', '140.44.4.24'), + ('sninfr2lnxwrk05', '140.44.4.25'), + ('sninfr2lnxwrk06', '140.44.4.26'), + ('sninfr2lnxwrk07', '140.44.4.27') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING AERODROME UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-133-aerodrome-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL AERODROME NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] AERODROME FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_aerodrome_134.py b/scripts/upgrade_aerodrome_134.py new file mode 100644 index 0000000..adc327b --- /dev/null +++ b/scripts/upgrade_aerodrome_134.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.34.12+k3s1' +ENCODED_VER = 'v1.34.12%2Bk3s1' + +masters = [ + ('sninfr2lnxctl01', '140.44.4.11'), + ('sninfr2lnxctl02', '140.44.4.12'), + ('sninfr2lnxctl03', '140.44.4.13') +] + +workers = [ + ('sninfr2lnxwrk01', '140.44.4.21'), + ('sninfr2lnxwrk02', '140.44.4.22'), + ('sninfr2lnxwrk03', '140.44.4.23'), + ('sninfr2lnxwrk04', '140.44.4.24'), + ('sninfr2lnxwrk05', '140.44.4.25'), + ('sninfr2lnxwrk06', '140.44.4.26'), + ('sninfr2lnxwrk07', '140.44.4.27') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING AERODROME UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-134-aerodrome-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL AERODROME NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] AERODROME FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_aerodrome_135.py b/scripts/upgrade_aerodrome_135.py new file mode 100644 index 0000000..45204b8 --- /dev/null +++ b/scripts/upgrade_aerodrome_135.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.35.9+k3s1' +ENCODED_VER = 'v1.35.9%2Bk3s1' + +masters = [ + ('sninfr2lnxctl01', '140.44.4.11'), + ('sninfr2lnxctl02', '140.44.4.12'), + ('sninfr2lnxctl03', '140.44.4.13') +] + +workers = [ + ('sninfr2lnxwrk01', '140.44.4.21'), + ('sninfr2lnxwrk02', '140.44.4.22'), + ('sninfr2lnxwrk03', '140.44.4.23'), + ('sninfr2lnxwrk04', '140.44.4.24'), + ('sninfr2lnxwrk05', '140.44.4.25'), + ('sninfr2lnxwrk06', '140.44.4.26'), + ('sninfr2lnxwrk07', '140.44.4.27') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING AERODROME UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-135-aerodrome-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL AERODROME NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] AERODROME FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_aerodrome_136.py b/scripts/upgrade_aerodrome_136.py new file mode 100644 index 0000000..81416b3 --- /dev/null +++ b/scripts/upgrade_aerodrome_136.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.36.5+k3s1' +ENCODED_VER = 'v1.36.5%2Bk3s1' + +masters = [ + ('sninfr2lnxctl01', '140.44.4.11'), + ('sninfr2lnxctl02', '140.44.4.12'), + ('sninfr2lnxctl03', '140.44.4.13') +] + +workers = [ + ('sninfr2lnxwrk01', '140.44.4.21'), + ('sninfr2lnxwrk02', '140.44.4.22'), + ('sninfr2lnxwrk03', '140.44.4.23'), + ('sninfr2lnxwrk04', '140.44.4.24'), + ('sninfr2lnxwrk05', '140.44.4.25'), + ('sninfr2lnxwrk06', '140.44.4.26'), + ('sninfr2lnxwrk07', '140.44.4.27') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING AERODROME UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-136-aerodrome-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Aerodrome Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL AERODROME NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] AERODROME FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_batch2.py b/scripts/upgrade_batch2.py new file mode 100644 index 0000000..32f14a0 --- /dev/null +++ b/scripts/upgrade_batch2.py @@ -0,0 +1,59 @@ +import paramiko +import time + +def upgrade_uptime_kuma(): + print("\n=======================================================") + print("[*] Upgrading Uptime Kuma to 2.5.9 on Aerodrome (140.44.4.11)...") + print("=======================================================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + # Get container name + _, out, _ = ssh.exec_command("kubectl get deployment uptime-kuma-2-1756668961 -n monitoring -o jsonpath='{.spec.template.spec.containers[0].name}'") + c_name = out.read().decode().strip() + print(f"[*] Container name: {c_name}") + + # Update image + cmd_up = f"kubectl set image deployment/uptime-kuma-2-1756668961 {c_name}=louislam/uptime-kuma:2.5.9 -n monitoring" + print(f"[*] Running: {cmd_up}") + ssh.exec_command(cmd_up) + + # Rollout status + stdin, stdout, stderr = ssh.exec_command("kubectl rollout status deployment/uptime-kuma-2-1756668961 -n monitoring --timeout=180s") + print(stdout.read().decode().strip()) + + _, out, _ = ssh.exec_command("kubectl get pods -n monitoring -l app.kubernetes.io/name=uptime-kuma -o wide") + print(out.read().decode().strip()) + ssh.close() + +def upgrade_semaphore(): + print("\n=======================================================") + print("[*] Upgrading Semaphore UI to v2.19.17 on Spaceport (140.44.4.41)...") + print("=======================================================") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + + # Get container name + _, out, _ = ssh.exec_command("kubectl get deployment semaphore -n infrastructure -o jsonpath='{.spec.template.spec.containers[0].name}'") + c_name = out.read().decode().strip() + print(f"[*] Container name: {c_name}") + + # Update image + cmd_up = f"kubectl set image deployment/semaphore {c_name}=semaphoreui/semaphore:v2.19.17 -n infrastructure" + print(f"[*] Running: {cmd_up}") + ssh.exec_command(cmd_up) + + # Rollout status + stdin, stdout, stderr = ssh.exec_command("kubectl rollout status deployment/semaphore -n infrastructure --timeout=180s") + print(stdout.read().decode().strip()) + + _, out, _ = ssh.exec_command("kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o wide") + print(out.read().decode().strip()) + ssh.close() + +if __name__ == '__main__': + upgrade_uptime_kuma() + upgrade_semaphore() + diff --git a/scripts/upgrade_coder.py b/scripts/upgrade_coder.py new file mode 100644 index 0000000..8d3dc53 --- /dev/null +++ b/scripts/upgrade_coder.py @@ -0,0 +1,24 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +print("[*] Upgrading Coder to v2.38.6 on Spaceport...") +run("kubectl set image deployment/coder coder=ghcr.io/coder/coder:v2.38.6 -n infrastructure") +run("kubectl rollout status deployment/coder -n infrastructure --timeout=180s") +run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=coder -o wide") +run("curl -k -s -I https://coder.snarfnet.net | head -n 8") + +ssh.close() + diff --git a/scripts/upgrade_grafana.py b/scripts/upgrade_grafana.py new file mode 100644 index 0000000..e87612b --- /dev/null +++ b/scripts/upgrade_grafana.py @@ -0,0 +1,23 @@ +import paramiko +import time + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + errors = err.read().decode().strip() + if res: + print(res) + if errors: + print("ERR:", errors) + +print("[*] Upgrading Grafana to 11.6.16 on Aerodrome...") +run("kubectl set image deployment/grafana grafana=docker.io/grafana/grafana:11.6.16 -n default") +run("kubectl rollout status deployment/grafana -n default --timeout=180s") +run("kubectl get pods -n default -l app.kubernetes.io/name=grafana -o wide") +ssh.close() + diff --git a/scripts/upgrade_homepage.py b/scripts/upgrade_homepage.py new file mode 100644 index 0000000..a63ec35 --- /dev/null +++ b/scripts/upgrade_homepage.py @@ -0,0 +1,22 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + errors = err.read().decode().strip() + if res: + print(res) + if errors: + print("ERR:", errors) + +print("[*] Upgrading Homepage Dashboard to v1.4.5 on Spaceport...") +run("kubectl set image deployment/homepage-2-1744068158 homepage-2-1744068158=ghcr.io/gethomepage/homepage:v1.4.5 -n infrastructure") +run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=180s") +run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=homepage -o wide") +ssh.close() + diff --git a/scripts/upgrade_jenkins.py b/scripts/upgrade_jenkins.py new file mode 100644 index 0000000..62b1223 --- /dev/null +++ b/scripts/upgrade_jenkins.py @@ -0,0 +1,23 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +print("[*] Upgrading Jenkins StatefulSet to 2.589-jdk21 on Spaceport...") +run("kubectl set image statefulset/jenkins jenkins=docker.io/jenkins/jenkins:2.589-jdk21 -n infrastructure") +run("kubectl rollout status statefulset/jenkins -n infrastructure --timeout=300s") +run("kubectl get pods -n infrastructure -l app.kubernetes.io/name=jenkins -o wide") + +ssh.close() + diff --git a/scripts/upgrade_nutserver.py b/scripts/upgrade_nutserver.py new file mode 100644 index 0000000..4b0ceaf --- /dev/null +++ b/scripts/upgrade_nutserver.py @@ -0,0 +1,72 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +IP = '14.4.4.44' + +print(f"=== UPGRADING NUTSERVER ({IP}) ===") + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect(IP, username='snarf', key_filename=KEY_FILE, timeout=10) + +upgrade_cmd = """ +echo "debconf debconf/frontend select Noninteractive" | sudo debconf-set-selections +echo "linux-base linux-base/removing-running-kernel boolean false" | sudo debconf-set-selections +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y +sudo apt-get autoremove -y +""" + +stdin, stdout, stderr = ssh.exec_command(upgrade_cmd) +exit_code = stdout.channel.recv_exit_status() +print(f"[*] apt dist-upgrade finished with exit code {exit_code}") + +# Check reboot +_, out, _ = ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") +reboot_needed = out.read().decode().strip() == 'yes' +print(f"[*] Reboot required: {reboot_needed}") + +if reboot_needed: + print("[*] Gracefully rebooting nutserver...") + try: + ssh.exec_command("sudo reboot") + except Exception: + pass + ssh.close() + + print("[*] Waiting 30s for reboot...") + time.sleep(30) + + reconnected = False + for attempt in range(25): + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(IP, username='snarf', key_filename=KEY_FILE, timeout=6) + reconnected = True + print(f"[*] nutserver reconnected on attempt {attempt+1}!") + break + except Exception: + time.sleep(4) + + if not reconnected: + print("[!] ERROR: nutserver failed to reconnect!") + sys.exit(1) + + _, out, _ = ssh.exec_command("systemctl is-active chrony || systemctl is-active chronyd") + print(" Chrony NTP:", out.read().decode().strip()) + _, out, _ = ssh.exec_command("systemctl is-active nut-server") + print(" NUT Server:", out.read().decode().strip()) + _, out, _ = ssh.exec_command("systemctl is-active zabbix-agent2") + print(" ZabbixAgent2:", out.read().decode().strip()) + ssh.close() +else: + ssh.close() + +print("[SUCCESS] nutserver OS upgraded cleanly!") + diff --git a/scripts/upgrade_portainer.py b/scripts/upgrade_portainer.py new file mode 100644 index 0000000..7d9e983 --- /dev/null +++ b/scripts/upgrade_portainer.py @@ -0,0 +1,62 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +IP = '140.44.4.71' + +print("=== UPGRADING PORTAINER SERVER & AGENT TO 2.45.3 ===") + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect(IP, username='snarf', key_filename=KEY_FILE, timeout=10) + +# 1. Backup portainer_data volume +print("[*] Backing up portainer_data volume...") +backup_cmd = """ +sudo mkdir -p /mnt/nfs_shares/backups/portainer +sudo tar -czf /mnt/nfs_shares/backups/portainer/portainer_data_$(date +%Y%m%d_%H%M%S).tar.gz -C /var/lib/docker/volumes/portainer_data _data +sudo ls -lh /mnt/nfs_shares/backups/portainer/ | tail -n 1 +""" +stdin, stdout, stderr = ssh.exec_command(backup_cmd) +print("Backup created:", stdout.read().decode().strip()) + +# 2. Recreate Portainer Server with 2.45.3 +print("[*] Recreating Portainer Server container with portainer/portainer-ee:2.45.3...") +upgrade_cmd = """ +sudo docker stop portainer +sudo docker rm portainer +sudo docker run -d \ + -p 8000:8000 \ + -p 9443:9443 \ + --name portainer \ + --restart=always \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v portainer_data:/data \ + -v /opt/portainer/admin_password:/tmp/admin_password \ + portainer/portainer-ee:2.45.3 \ + --admin-password-file /tmp/admin_password +""" +stdin, stdout, stderr = ssh.exec_command(upgrade_cmd) +stdout.channel.recv_exit_status() +print("Portainer Server container recreated!") + +# 3. Update Portainer Agent Swarm Service +print("[*] Updating Portainer Agent Swarm Service to 2.45.3...") +agent_cmd = "sudo docker service update --image portainer/agent:2.45.3 portainer_agent" +stdin, stdout, stderr = ssh.exec_command(agent_cmd) +stdout.channel.recv_exit_status() +print(stdout.read().decode()) + +time.sleep(5) + +# 4. Verify Portainer Server & Agent +stdin, stdout, stderr = ssh.exec_command('sudo docker ps -f name=portainer; sudo docker service ls | grep portainer') +print("\n=== STATUS ===") +print(stdout.read().decode()) + +ssh.close() +print("[SUCCESS] Portainer upgraded to 2.45.3!") + diff --git a/scripts/upgrade_remaining_workers.py b/scripts/upgrade_remaining_workers.py new file mode 100644 index 0000000..4638e41 --- /dev/null +++ b/scripts/upgrade_remaining_workers.py @@ -0,0 +1,107 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.11' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.31.4+k3s1' +ENCODED_VER = 'v1.31.4%2Bk3s1' + +workers = [ + ('sninfr2lnxwrk03', '140.44.4.23'), + ('sninfr2lnxwrk04', '140.44.4.24'), + ('sninfr2lnxwrk05', '140.44.4.25'), + ('sninfr2lnxwrk06', '140.44.4.26'), + ('sninfr2lnxwrk07', '140.44.4.27') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print("=== CHECKING CURRENT NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +# Ensure wrk03 is uncordoned first if stuck +master_ssh.exec_command('kubectl uncordon sninfr2lnxwrk03') + +for node_name, worker_ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading worker {node_name} ({worker_ip}) to {TARGET_VERSION}...") + print(f"=======================================================") + + # 1. Cordon worker + print(f"[*] Cordoning {node_name}...") + master_ssh.exec_command(f'kubectl cordon {node_name}') + + # 2. Drain worker safely + print(f"[*] Draining {node_name} (ignoring daemonsets)...") + _, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {node_name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + print("Drain:", stdout.read().decode().strip()[:200]) + + # 3. Connect to worker and update binary + print(f"[*] Connecting directly to worker {worker_ip}...") + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(worker_ip, username='root', key_filename=KEY_FILE, timeout=10) + + # Pre-seed debconf and ensure open-iscsi + w_ssh.exec_command('echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections; echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections') + + dl_cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + print(f"[*] Downloading K3s binary & restarting k3s-agent...") + stdin, stdout, stderr = w_ssh.exec_command(dl_cmd) + stdout.channel.recv_exit_status() + + # Wait for k3s-agent to be active + for _ in range(15): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + status = out.read().decode().strip() + if status == 'active': + print(f"[*] k3s-agent is active on {node_name}!") + break + time.sleep(2) + + w_ssh.close() + + # 4. Uncordon worker + print(f"[*] Uncordoning {node_name}...") + master_ssh.exec_command(f'kubectl uncordon {node_name}') + + # 5. Wait for node Ready in Kubernetes + print(f"[*] Verifying node Ready state in Kubernetes...") + for _ in range(25): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {node_name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] {node_name} is fully Ready!") + break + time.sleep(4) + + # 6. Settle Longhorn storage replicas + print(f"[*] Settling Longhorn storage replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL CLUSTER NODE STATUS ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +print("=== LONGHORN VOLUME HEALTH ===") +_, out, _ = master_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system') +print(out.read().decode()) + +master_ssh.close() +print("\n[SUCCESS] ALL AERODROME WORKERS UPGRADED TO v1.31.4+k3s1!") + diff --git a/scripts/upgrade_spaceport_132.py b/scripts/upgrade_spaceport_132.py new file mode 100644 index 0000000..a66b950 --- /dev/null +++ b/scripts/upgrade_spaceport_132.py @@ -0,0 +1,143 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.41' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.32.13+k3s1' +ENCODED_VER = 'v1.32.13%2Bk3s1' + +masters = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44') +] + +workers = [ + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-132-spaceport-{int(time.time())}') + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + stdout.channel.recv_exit_status() + + # Wait for active + for _ in range(25): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + # Wait for Ready in Kube + for _ in range(30): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + stdout.channel.recv_exit_status() + + for _ in range(25): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(30): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL SPACEPORT NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +print("=== FINAL LONGHORN VOLUMES ===") +_, out, _ = master_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_spaceport_133.py b/scripts/upgrade_spaceport_133.py new file mode 100644 index 0000000..cabb6ae --- /dev/null +++ b/scripts/upgrade_spaceport_133.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.41' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.33.13+k3s2' +ENCODED_VER = 'v1.33.13%2Bk3s2' + +masters = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44') +] + +workers = [ + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-133-spaceport-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL SPACEPORT NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_spaceport_134.py b/scripts/upgrade_spaceport_134.py new file mode 100644 index 0000000..9cac2a3 --- /dev/null +++ b/scripts/upgrade_spaceport_134.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.41' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.34.12+k3s1' +ENCODED_VER = 'v1.34.12%2Bk3s1' + +masters = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44') +] + +workers = [ + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-134-spaceport-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL SPACEPORT NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_spaceport_135.py b/scripts/upgrade_spaceport_135.py new file mode 100644 index 0000000..8739660 --- /dev/null +++ b/scripts/upgrade_spaceport_135.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.41' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.35.9+k3s1' +ENCODED_VER = 'v1.35.9%2Bk3s1' + +masters = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44') +] + +workers = [ + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-135-spaceport-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL SPACEPORT NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_spaceport_136.py b/scripts/upgrade_spaceport_136.py new file mode 100644 index 0000000..a0c0bd8 --- /dev/null +++ b/scripts/upgrade_spaceport_136.py @@ -0,0 +1,148 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.41' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +TARGET_VERSION = 'v1.36.5+k3s1' +ENCODED_VER = 'v1.36.5%2Bk3s1' + +masters = [ + ('sninfr0lnxctl01', '140.44.4.41'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl04', '140.44.4.44') +] + +workers = [ + ('sninfr0lnxwrk01', '140.44.4.51'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk06', '140.44.4.56') +] + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +print(f"=== INITIATING SPACEPORT UPGRADE TO {TARGET_VERSION} ===") + +# 1. Save etcd snapshot +print("[*] Saving pre-upgrade etcd snapshot...") +stdin, stdout, stderr = master_ssh.exec_command(f'k3s etcd-snapshot save --name pre-136-spaceport-{int(time.time())}') +stdout.channel.recv_exit_status() +print("[*] Etcd snapshot complete.") + +# 2. Upgrade Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s + """ + stdin, stdout, stderr = m_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = m_ssh.exec_command('systemctl is-active k3s') + if out.read().decode().strip() == 'active': + print(f"[*] k3s service is active on {name}!") + break + time.sleep(3) + + m_ssh.close() + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Master {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling etcd consensus (15s)...") + time.sleep(15) + +# 3. Upgrade Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport Worker {name} ({ip})...") + print(f"=======================================================") + + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + print(f"[*] Draining {name}...") + stdin, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + stdout.channel.recv_exit_status() + + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + cmd = f""" + set -e + curl -sfL "https://github.com/k3s-io/k3s/releases/download/{ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new + test $(stat -c%s /usr/local/bin/k3s.new) -gt 60000000 + chmod +x /usr/local/bin/k3s.new + mv /usr/local/bin/k3s.new /usr/local/bin/k3s + systemctl restart k3s-agent + """ + stdin, stdout, stderr = w_ssh.exec_command(cmd) + exit_code = stdout.channel.recv_exit_status() + if exit_code != 0: + print(f"[ERROR] Failed to upgrade binary on worker {name}: {stderr.read().decode()}") + sys.exit(1) + + for _ in range(30): + _, out, _ = w_ssh.exec_command('systemctl is-active k3s-agent') + if out.read().decode().strip() == 'active': + print(f"[*] k3s-agent is active on {name}!") + break + time.sleep(3) + + w_ssh.close() + + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + for _ in range(35): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] Worker {name} is Ready!") + break + time.sleep(4) + + print("[*] Settling Longhorn replicas (15s)...") + time.sleep(15) + +print("\n=== FINAL SPACEPORT NODES ===") +_, out, _ = master_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +master_ssh.close() +print(f"\n[SUCCESS] SPACEPORT FULLY UPGRADED TO {TARGET_VERSION}!") + diff --git a/scripts/upgrade_spaceport_os.py b/scripts/upgrade_spaceport_os.py new file mode 100644 index 0000000..ab0c809 --- /dev/null +++ b/scripts/upgrade_spaceport_os.py @@ -0,0 +1,195 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +MASTER_IP = '140.44.4.41' +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' + +workers = [ + ('sninfr0lnxwrk06', '140.44.4.56'), + ('sninfr0lnxwrk05', '140.44.4.55'), + ('sninfr0lnxwrk04', '140.44.4.54'), + ('sninfr0lnxwrk03', '140.44.4.53'), + ('sninfr0lnxwrk02', '140.44.4.52'), + ('sninfr0lnxwrk01', '140.44.4.51') +] + +masters = [ + ('sninfr0lnxctl04', '140.44.4.44'), + ('sninfr0lnxctl03', '140.44.4.43'), + ('sninfr0lnxctl02', '140.44.4.42'), + ('sninfr0lnxctl01', '140.44.4.41') +] + +upgrade_cmd = """ +echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections +echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y +apt-get autoremove -y +""" + +master_ssh = paramiko.SSHClient() +master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10) + +# Phase 1: Workers +for name, ip in workers: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport worker {name} ({ip})...") + print(f"=======================================================") + + # 1. Cordon + print(f"[*] Cordoning {name}...") + master_ssh.exec_command(f'kubectl cordon {name}') + + # 2. Drain + print(f"[*] Draining {name} (ignoring daemonsets)...") + _, stdout, stderr = master_ssh.exec_command( + f'kubectl drain {name} --ignore-daemonsets --delete-emptydir-data --force --timeout=120s' + ) + print("Drain:", stdout.read().decode().strip()[:200]) + + # 3. Connect to worker + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + print(f"[*] Running dist-upgrade on {name}...") + stdin, stdout, stderr = w_ssh.exec_command(upgrade_cmd) + exit_code = stdout.channel.recv_exit_status() + print(f"[*] Upgrade finished with exit code {exit_code}") + + # Check reboot + _, out, _ = w_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") + reboot_needed = out.read().decode().strip() == 'yes' + print(f"[*] Reboot required: {reboot_needed}") + + if reboot_needed: + print(f"[*] Rebooting {name}...") + try: + w_ssh.exec_command("reboot") + except Exception: + pass + w_ssh.close() + + print("[*] Waiting 30s for reboot...") + time.sleep(30) + + reconnected = False + for attempt in range(25): + try: + w_ssh = paramiko.SSHClient() + w_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + w_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6) + reconnected = True + print(f"[*] {name} reconnected successfully on attempt {attempt+1}!") + break + except Exception: + time.sleep(4) + + if not reconnected: + print(f"[!] ERROR: {name} failed to reconnect!") + sys.exit(1) + + _, out, _ = w_ssh.exec_command("systemctl is-active k3s-agent") + print(f" K3sAgent: {out.read().decode().strip()}") + w_ssh.close() + else: + w_ssh.close() + + # 4. Uncordon + print(f"[*] Uncordoning {name}...") + master_ssh.exec_command(f'kubectl uncordon {name}') + + # 5. Verify Ready + print(f"[*] Verifying node Ready state in Kubernetes...") + for _ in range(25): + _, out, _ = master_ssh.exec_command( + f"kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'" + ) + if out.read().decode().strip() == 'True': + print(f"[OK] {name} is fully Ready!") + break + time.sleep(4) + + # 6. Settle Longhorn + print(f"[*] Settling Longhorn storage replicas (15s)...") + time.sleep(15) + +# Phase 2: Masters +for name, ip in masters: + print(f"\n=======================================================") + print(f"[*] Upgrading Spaceport master {name} ({ip})...") + print(f"=======================================================") + + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=10) + + print(f"[*] Running dist-upgrade on {name}...") + stdin, stdout, stderr = m_ssh.exec_command(upgrade_cmd) + exit_code = stdout.channel.recv_exit_status() + print(f"[*] Upgrade finished with exit code {exit_code}") + + _, out, _ = m_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") + reboot_needed = out.read().decode().strip() == 'yes' + print(f"[*] Reboot required: {reboot_needed}") + + if reboot_needed: + print(f"[*] Rebooting {name}...") + try: + m_ssh.exec_command("reboot") + except Exception: + pass + m_ssh.close() + + print("[*] Waiting 30s for reboot...") + time.sleep(30) + + reconnected = False + for attempt in range(25): + try: + m_ssh = paramiko.SSHClient() + m_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + m_ssh.connect(ip, username='root', key_filename=KEY_FILE, timeout=6) + reconnected = True + print(f"[*] {name} reconnected successfully on attempt {attempt+1}!") + break + except Exception: + time.sleep(4) + + if not reconnected: + print(f"[!] ERROR: {name} failed to reconnect!") + sys.exit(1) + + _, out, _ = m_ssh.exec_command("systemctl is-active k3s") + print(f" K3s: {out.read().decode().strip()}") + m_ssh.close() + else: + m_ssh.close() + + print(f"[*] Settling etcd consensus (15s)...") + time.sleep(15) + +master_ssh.close() + +# Final Verification from 140.44.4.41 +final_ssh = paramiko.SSHClient() +final_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +final_ssh.connect('140.44.4.41', username='root', key_filename=KEY_FILE, timeout=10) + +print("\n=== FINAL SPACEPORT NODES ===") +_, out, _ = final_ssh.exec_command('kubectl get nodes') +print(out.read().decode()) + +print("=== FINAL LONGHORN VOLUMES ===") +_, out, _ = final_ssh.exec_command('kubectl get volumes.longhorn.io -n longhorn-system') +print(out.read().decode()) + +final_ssh.close() +print("[SUCCESS] Spaceport rolling OS upgrade completed flawlessly!") + diff --git a/scripts/upgrade_swarm_cluster.py b/scripts/upgrade_swarm_cluster.py new file mode 100644 index 0000000..5bdfacf --- /dev/null +++ b/scripts/upgrade_swarm_cluster.py @@ -0,0 +1,97 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet' +swarm_nodes = [ + ('snshpd4lnxswm04', '140.44.4.74'), + ('snshpd4lnxswm03', '140.44.4.73'), + ('snshpd4lnxswm02', '140.44.4.72'), + ('snshpd4lnxswm01', '140.44.4.71') +] + +upgrade_cmd = """ +echo "debconf debconf/frontend select Noninteractive" | sudo debconf-set-selections +echo "linux-base linux-base/removing-running-kernel boolean false" | sudo debconf-set-selections +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y +sudo apt-get autoremove -y +""" + +for name, ip in swarm_nodes: + print(f"\n=======================================================") + print(f"[*] Upgrading Docker Swarm node {name} ({ip})...") + print(f"=======================================================") + + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='snarf', key_filename=KEY_FILE, timeout=10) + + stdin, stdout, stderr = ssh.exec_command(upgrade_cmd) + exit_code = stdout.channel.recv_exit_status() + print(f"[*] apt dist-upgrade finished with exit code {exit_code}") + + # Check reboot + _, out, _ = ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'") + reboot_needed = out.read().decode().strip() == 'yes' + print(f"[*] Reboot required: {reboot_needed}") + + if reboot_needed: + print(f"[*] Gracefully rebooting {name}...") + try: + ssh.exec_command("sudo reboot") + except Exception: + pass + ssh.close() + + print("[*] Waiting 30s for reboot...") + time.sleep(30) + + reconnected = False + for attempt in range(25): + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username='snarf', key_filename=KEY_FILE, timeout=6) + reconnected = True + print(f"[*] {name} reconnected successfully on attempt {attempt+1}!") + break + except Exception: + time.sleep(4) + + if not reconnected: + print(f"[!] ERROR: {name} failed to reconnect!") + sys.exit(1) + + _, out, _ = ssh.exec_command("sudo systemctl is-active docker") + print(f" Docker: {out.read().decode().strip()}") + ssh.close() + else: + ssh.close() + + # Settle Swarm consensus + print(f"[*] Settling Swarm Raft consensus (15s)...") + time.sleep(15) + +# Verify from swm01 +final_ssh = paramiko.SSHClient() +final_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +final_ssh.connect('140.44.4.71', username='snarf', key_filename=KEY_FILE, timeout=10) +_, out, _ = final_ssh.exec_command('sudo docker node ls') +print("\n=== FINAL DOCKER SWARM NODES ===") +print(out.read().decode()) + +_, out, _ = final_ssh.exec_command('sudo docker service ls') +print("=== DOCKER SERVICES ===") +print(out.read().decode()) + +_, out, _ = final_ssh.exec_command('ip addr show | grep 140.44.4') +print("=== KEEPALIved VIP 140.44.4.70 ===") +print(out.read().decode()) +final_ssh.close() + +print("[SUCCESS] All Docker Swarm nodes upgraded to latest Docker Engine and packages!") + diff --git a/scripts/upgrade_zabbix.py b/scripts/upgrade_zabbix.py new file mode 100644 index 0000000..f51146e --- /dev/null +++ b/scripts/upgrade_zabbix.py @@ -0,0 +1,35 @@ +import paramiko + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10) + +def run(cmd): + print(f"=== {cmd} ===") + _, out, err = ssh.exec_command(cmd) + res = out.read().decode().strip() + if res: + print(res) + e = err.read().decode().strip() + if e: + print("ERR:", e) + +print("[*] Upgrading Zabbix stack to ubuntu-7.0.32 on Spaceport...") + +run("kubectl set image deployment/zabbix-zabbix-server zabbix-server=zabbix/zabbix-server-pgsql:ubuntu-7.0.32 zabbix-agent=zabbix/zabbix-agent2:ubuntu-7.0.32 -n monitoring") +run("kubectl set image deployment/zabbix-zabbix-web zabbix-web=zabbix/zabbix-web-nginx-pgsql:ubuntu-7.0.32 -n monitoring") +run("kubectl set image deployment/zabbix-zabbix-webservice zabbix-webservice=zabbix/zabbix-web-service:ubuntu-7.0.32 -n monitoring") + +print("[*] Awaiting rollouts...") +run("kubectl rollout status deployment/zabbix-zabbix-server -n monitoring --timeout=180s") +run("kubectl rollout status deployment/zabbix-zabbix-web -n monitoring --timeout=180s") +run("kubectl rollout status deployment/zabbix-zabbix-webservice -n monitoring --timeout=180s") + +print("[*] Checking pod statuses...") +run("kubectl get pods -n monitoring -o wide") + +print("[*] Verifying HTTP endpoint...") +run("curl -s -I http://140.44.4.51:8888 | head -n 8") + +ssh.close() + diff --git a/snarfcode_update.bundle b/snarfcode_update.bundle index 1b24ada..412d77b 100644 Binary files a/snarfcode_update.bundle and b/snarfcode_update.bundle differ