feat: add scripts for k3s cluster upgrade and health checks via AWX and SSH

scripts/check_13_boot.py

@@ -0,0 +1,13 @@
+import paramiko, sys
+sys.stdout.reconfigure(encoding='utf-8')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+try:
+    ssh.connect('140.44.4.13', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    stdin, stdout, stderr = ssh.exec_command('hostname; uptime; k3s --version')
+    print("SSH SUCCESS TO .13:\n", stdout.read().decode())
+    ssh.close()
+except Exception as e:
+    print(f"SSH Failed: {e}")
+

scripts/check_job_115.py

@@ -0,0 +1,21 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Job
+job = Job.objects.get(id=115)
+print(f"Job [115] for '{job.name}': Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+events = job.job_events.order_by('id')
+for e in events:
+    if e.stdout:
+        print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job_121.py

@@ -0,0 +1,21 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Job
+job = Job.objects.get(id=121)
+print(f"Job [121] for '{job.name}': Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+events = job.job_events.order_by('id')
+for e in events:
+    if e.stdout:
+        print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job_124.py

@@ -0,0 +1,21 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.12', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Job
+job = Job.objects.get(id=124)
+print(f"Job [124] for '{job.name}': Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+events = job.job_events.order_by('id')
+for e in events:
+    if e.stdout:
+        print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/check_job_127.py

@@ -0,0 +1,21 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.12', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Job
+job = Job.objects.get(id=127)
+print(f"Job [127] for '{job.name}': Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+events = job.job_events.order_by('id')
+for e in events:
+    if e.stdout:
+        print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/launch_upgrade_step.py

@@ -0,0 +1,54 @@
+import paramiko, sys, time
+
+version = sys.argv[1] if len(sys.argv) > 1 else "v1.27.16+k3s1"
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = f"""
+from awx.main.models import Project, JobTemplate, Job
+import time, json
+
+# 1. Project Update
+p = Project.objects.get(name='Snarfnet Infrastructure Playbooks')
+update = p.update()
+for _ in range(15):
+    update.refresh_from_db()
+    if update.status in ['successful', 'failed', 'error']:
+        break
+    time.sleep(2)
+
+print(f"Project update [{{update.id}}]: {{update.status}}")
+
+# 2. Launch Job Template
+jt = JobTemplate.objects.get(name='Aerodrome K3s Control Plane Upgrade')
+job = jt.create_unified_job(extra_vars=json.dumps({{'target_k3s_version': '{version}'}}))
+job.signal_start()
+print(f"Launched Upgrade Job [{{job.id}}] targeting version '{version}': Status={{job.status}}")
+
+for _ in range(120):
+    job.refresh_from_db()
+    if job.status in ['successful', 'failed', 'error', 'canceled']:
+        break
+    time.sleep(3)
+
+print(f"Job [{{job.id}}] finished with Status={{job.status}}, Elapsed={{job.elapsed}}s, Failed={{job.failed}}")
+
+print('\\n=== PLAY RECAP ===')
+for e in job.job_events.filter(event='playbook_on_stats'):
+    print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+
+for line in iter(stdout.readline, ""):
+    print(line, end="")
+
+# Check kubectl get nodes
+print("\n=== CLUSTER STATUS AFTER STEP ===")
+stdin, stdout, stderr = ssh.exec_command("kubectl get nodes -o wide")
+print(stdout.read().decode())
+
+ssh.close()

scripts/register_upgrade_jt.py

@@ -0,0 +1,42 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Project, Inventory, Credential, JobTemplate
+
+# 1. Update Project
+p = Project.objects.get(name='Snarfnet Infrastructure Playbooks')
+update = p.update()
+print(f"Triggered Project update [{update.id}]")
+
+inv = Inventory.objects.get(name='Snarfnet Master Inventory')
+cred_root = Credential.objects.get(name='Snarfnet Master SSH Key (root)')
+
+# 2. Register Job Template
+jt, created = JobTemplate.objects.get_or_create(
+    name='Aerodrome K3s Control Plane Upgrade',
+    defaults={
+        'project': p,
+        'inventory': inv,
+        'playbook': 'ansible/playbooks/upgrade_k3s_aerodrome.yml',
+        'ask_variables_on_launch': True,
+        'description': 'Sequential minor release upgrade of Aerodrome K3s control plane with etcd quorum check'
+    }
+)
+if not created:
+    jt.playbook = 'ansible/playbooks/upgrade_k3s_aerodrome.yml'
+    jt.ask_variables_on_launch = True
+    jt.save()
+
+jt.credentials.add(cred_root)
+print(f"Job Template '{jt.name}' (id={jt.id}, created={created}) registered.")
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print(stdout.read().decode())
+ssh.close()
+

scripts/run_aerodrome_precheck.py

@@ -0,0 +1,45 @@
+import paramiko, time
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+py_code = """
+from awx.main.models import Project, JobTemplate, Job
+import time
+
+p = Project.objects.get(name='Snarfnet Infrastructure Playbooks')
+update = p.update()
+for _ in range(15):
+    update.refresh_from_db()
+    if update.status in ['successful', 'failed', 'error']:
+        break
+    time.sleep(2)
+
+print(f"Project update [{update.id}]: {update.status}")
+
+jt = JobTemplate.objects.get(name='Aerodrome K3s Pre-Upgrade Check')
+job = jt.create_unified_job()
+job.signal_start()
+print(f"Launched Job [{job.id}] for '{jt.name}': Status={job.status}")
+
+for _ in range(30):
+    job.refresh_from_db()
+    if job.status in ['successful', 'failed', 'error', 'canceled']:
+        break
+    time.sleep(2)
+
+print(f"Job [{job.id}] finished with Status={job.status}, Elapsed={job.elapsed}s, Failed={job.failed}")
+
+print('\\n=== PLAY RECAP ===')
+for e in job.job_events.filter(event='playbook_on_stats'):
+    print(e.stdout)
+"""
+
+cmd = f"kubectl exec -i -n awx -c awx-task deployment/awx-task -- awx-manage shell_plus --quiet << 'EOF'\n{py_code}\nEOF"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+
+for line in iter(stdout.readline, ""):
+    print(line, end="")
+
+ssh.close()

scripts/test_zabbix_13.py

@@ -0,0 +1,12 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmd = "kubectl exec -n monitoring deployment/zabbix-zabbix-server -c zabbix-server -- zabbix_get -s 140.44.4.13 -k system.uptime"
+stdin, stdout, stderr = ssh.exec_command(cmd)
+print("OUT:", stdout.read().decode())
+print("ERR:", stderr.read().decode())
+ssh.close()
+

scripts/verify_cluster_health.py

@@ -0,0 +1,20 @@
+import paramiko
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+cmds = [
+    "kubectl get pods -n kube-system",
+    "kubectl get pods -n awx",
+    "kubectl get pods -n infra",
+    "kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness | head -n 15"
+]
+
+for cmd in cmds:
+    print(f"\n=== {cmd} ===")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+
+ssh.close()
+