From 93e13c748219dd1f7b5847bf75c508e9b7faf981 Mon Sep 17 00:00:00 2001 From: Snarf Date: Wed, 4 Nov 2026 21:35:52 -0500 Subject: [PATCH] feat: add scripts for testing and automating Rancher upgrades and chart checks --- scripts/check_chart_kubeversions.py | 1 + scripts/check_psp.py | 1 + scripts/step_rancher.py | 92 +++++++++++++++++++++++++++++ scripts/test_dryrun_215.py | 1 + scripts/test_dryrun_285.py | 1 + scripts/test_dryrun_kubeversion.py | 1 + scripts/test_dryrun_resetvalues.py | 1 + scripts/test_patch_chart.py | 1 + scripts/upgrade_rancher_step.py | 1 + 9 files changed, 100 insertions(+) create mode 100644 scripts/step_rancher.py diff --git a/scripts/check_chart_kubeversions.py b/scripts/check_chart_kubeversions.py index 660ebcb..aca91eb 100644 --- a/scripts/check_chart_kubeversions.py +++ b/scripts/check_chart_kubeversions.py @@ -19,3 +19,4 @@ for ver in versions: print(f"Rancher {ver}: {out}") client.close() + diff --git a/scripts/check_psp.py b/scripts/check_psp.py index bfbd431..715276d 100644 --- a/scripts/check_psp.py +++ b/scripts/check_psp.py @@ -18,3 +18,4 @@ stdin, stdout, stderr = client.exec_command(test_cmd) out = stdout.read().decode('utf-8') print(out) client.close() + diff --git a/scripts/step_rancher.py b/scripts/step_rancher.py new file mode 100644 index 0000000..3148dc8 --- /dev/null +++ b/scripts/step_rancher.py @@ -0,0 +1,92 @@ +import paramiko +import sys +import time + +sys.stdout.reconfigure(encoding='utf-8') + +if len(sys.argv) < 2: + print("Usage: python scripts/step_rancher.py ") + sys.exit(1) + +target_version = sys.argv[1] +key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet" +master_ip = "140.44.4.11" + +print("==================================================") +print(f"Rancher Upgrade Hop: Target Version {target_version}") +print("==================================================") + +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + +def run_cmd(cmd, check=True): + print(f"\n>>> Running: {cmd}") + stdin, stdout, stderr = client.exec_command(f"KUBECONFIG=/etc/rancher/k3s/k3s.yaml {cmd}") + out = stdout.read().decode('utf-8') + err = stderr.read().decode('utf-8') + exit_status = stdout.channel.recv_exit_status() + if out: + print(out.strip()) + if err: + print(f"STDERR: {err.strip()}") + if check and exit_status != 0: + raise RuntimeError(f"Command failed with exit code {exit_status}") + return out, err, exit_status + +try: + client.connect(master_ip, username="root", key_filename=key_path, timeout=15) + + # 1. Take snapshot + snap_name = f"pre-rancher-{target_version.replace('.', '-')}-{int(time.time())}" + print(f"\n--- 1. Taking etcd snapshot: {snap_name} ---") + run_cmd(f"k3s etcd-snapshot save --name {snap_name}") + + # 2. Pull chart and patch kubeVersion + print(f"\n--- 2. Pulling and preparing chart {target_version} ---") + setup_cmd = f""" +rm -rf /tmp/rancher-hop && mkdir -p /tmp/rancher-hop && cd /tmp/rancher-hop +helm pull rancher-stable/rancher --version {target_version} --untar +sed -i '/kubeVersion:/d' /tmp/rancher-hop/rancher/Chart.yaml +""" + run_cmd(setup_cmd) + + # 3. Perform Helm upgrade + print(f"\n--- 3. Executing Helm upgrade to {target_version} ---") + upgrade_cmd = ( + f"helm upgrade rancher /tmp/rancher-hop/rancher " + f"--namespace cattle-system " + f"--set hostname=aerodrome.snarfnet.net " + f"--set ingress.tls.source=secret" + ) + run_cmd(upgrade_cmd) + + # 4. Wait for deployment rollout + print("\n--- 4. Waiting for deployment/rancher rollout status (timeout 400s) ---") + run_cmd("kubectl -n cattle-system rollout status deployment/rancher --timeout=400s") + + # 5. Check cattle-system pods + print("\n--- 5. Checking cattle-system pods ---") + run_cmd("kubectl -n cattle-system get pods -o wide") + + # 6. Check webhook rollout if webhook deployment exists + print("\n--- 6. Checking rancher-webhook status ---") + run_cmd("kubectl -n cattle-system get pods -l app=rancher-webhook", check=False) + + # 7. Check ingress / HTTP response + print("\n--- 7. Checking Rancher endpoint ---") + time.sleep(5) + run_cmd("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10", check=False) + + # 8. Check helm release status + print("\n--- 8. Helm release status ---") + run_cmd("helm list -n cattle-system") + + client.close() + print(f"\n==================================================") + print(f"SUCCESS: Rancher upgraded to {target_version} successfully!") + print(f"==================================================") + +except Exception as ex: + print(f"\nFAILED: {ex}") + sys.exit(1) + diff --git a/scripts/test_dryrun_215.py b/scripts/test_dryrun_215.py index 420f9cf..a7067cc 100644 --- a/scripts/test_dryrun_215.py +++ b/scripts/test_dryrun_215.py @@ -21,3 +21,4 @@ print("STDOUT:", out[:500]) print("STDERR:", err) client.close() + diff --git a/scripts/test_dryrun_285.py b/scripts/test_dryrun_285.py index 78ebf76..80effc4 100644 --- a/scripts/test_dryrun_285.py +++ b/scripts/test_dryrun_285.py @@ -24,3 +24,4 @@ print("STDOUT:", out[:400]) print("STDERR:", err) client.close() + diff --git a/scripts/test_dryrun_kubeversion.py b/scripts/test_dryrun_kubeversion.py index 206747f..eb850e4 100644 --- a/scripts/test_dryrun_kubeversion.py +++ b/scripts/test_dryrun_kubeversion.py @@ -21,3 +21,4 @@ print("STDOUT:", out[:300]) print("STDERR:", err) client.close() + diff --git a/scripts/test_dryrun_resetvalues.py b/scripts/test_dryrun_resetvalues.py index 9c1442d..61a1a4d 100644 --- a/scripts/test_dryrun_resetvalues.py +++ b/scripts/test_dryrun_resetvalues.py @@ -22,3 +22,4 @@ print("STDOUT:", out[:500]) print("STDERR:", err) client.close() + diff --git a/scripts/test_patch_chart.py b/scripts/test_patch_chart.py index 193c078..b5d9228 100644 --- a/scripts/test_patch_chart.py +++ b/scripts/test_patch_chart.py @@ -23,3 +23,4 @@ print("STDOUT:", out[:400]) print("STDERR:", err) client.close() + diff --git a/scripts/upgrade_rancher_step.py b/scripts/upgrade_rancher_step.py index 2cf3aee..4703dea 100644 --- a/scripts/upgrade_rancher_step.py +++ b/scripts/upgrade_rancher_step.py @@ -65,3 +65,4 @@ try: except Exception as ex: print(f"\nFAILED: {ex}") sys.exit(1) +