From fc42bd6f94f1229720de64e30392adce893e6f15 Mon Sep 17 00:00:00 2001 From: Snarf Date: Sun, 8 Nov 2026 09:28:06 -0500 Subject: [PATCH] feat: add DNS setup blueprint and infrastructure automation scripts --- docs/DNS_Setup_Plan.md | 2 + scripts/apply_bios_tuning.py | 51 ++++++++++++++++++ scripts/audit_harvester_hardware.py | 75 +++++++++++++++++++++++++++ scripts/audit_harvester_live_nodes.py | 24 +++++++++ scripts/check_gpus_idrac.py | 37 +++++++++++++ scripts/check_node4_specs.py | 28 ++++++++++ scripts/check_nodes2_3.py | 22 ++++++++ scripts/check_redfish_node4.py | 34 ++++++++++++ scripts/inspect_harvester_live.py | 11 ++++ scripts/post_switch_healthcheck.py | 20 +++++++ scripts/test_racadm_queries.py | 31 +++++++++++ 11 files changed, 335 insertions(+) create mode 100644 scripts/apply_bios_tuning.py create mode 100644 scripts/audit_harvester_hardware.py create mode 100644 scripts/audit_harvester_live_nodes.py create mode 100644 scripts/check_gpus_idrac.py create mode 100644 scripts/check_node4_specs.py create mode 100644 scripts/check_nodes2_3.py create mode 100644 scripts/check_redfish_node4.py create mode 100644 scripts/inspect_harvester_live.py create mode 100644 scripts/post_switch_healthcheck.py create mode 100644 scripts/test_racadm_queries.py diff --git a/docs/DNS_Setup_Plan.md b/docs/DNS_Setup_Plan.md index 5ce3639..b561431 100644 --- a/docs/DNS_Setup_Plan.md +++ b/docs/DNS_Setup_Plan.md @@ -180,3 +180,5 @@ We will define firewall groups on the UDM Pro: * Secondary: `140.44.4.65` * Set UDM Pro WAN DNS to `140.44.4.35` & `140.44.4.65` (or Quad9 fallback). * Apply LAN IN firewall rule matrix on UDM Pro to enforce strict VLAN isolation. + +snaisvr44hrvstr0x - AppropriateBeav3r39! \ No newline at end of file diff --git a/scripts/apply_bios_tuning.py b/scripts/apply_bios_tuning.py new file mode 100644 index 0000000..16fba73 --- /dev/null +++ b/scripts/apply_bios_tuning.py @@ -0,0 +1,51 @@ +import paramiko +import time + +targets = [ + { + "name": "Node 1 (140.44.44.121)", + "ip": "140.44.44.121", + "settings": [ + ("bios.IntegratedDevices.SriovGlobalEnable", "Enabled") + ] + }, + { + "name": "Node 4 (140.44.44.124)", + "ip": "140.44.44.124", + "settings": [ + ("bios.SysProfileSettings.SysProfile", "PerfOptimized") + ] + } +] + +for t in targets: + print(f"\n==========================================") + print(f"Applying BIOS settings to {t['name']}...") + try: + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(t['ip'], username='root', password='Katzir476!', timeout=15) + + for key, val in t['settings']: + cmd = f"racadm set {key} {val}" + stdin, stdout, stderr = c.exec_command(cmd) + out = stdout.read().decode().strip() + print(f" racadm set {key} {val}:\n {out}") + time.sleep(1) + + # Create BIOS job + stdin, stdout, stderr = c.exec_command("racadm jobqueue create BIOS.Setup.1-1") + job_out = stdout.read().decode().strip() + print(f" racadm jobqueue create BIOS.Setup.1-1:\n {job_out}") + + # View job queue + stdin, stdout, stderr = c.exec_command("racadm jobqueue view") + q_out = stdout.read().decode().strip() + print(f" Job Queue:\n {q_out[:200]}") + + c.close() + except Exception as e: + print(f" Error on {t['name']}: {e}") + +print("\nDone applying BIOS settings!") + diff --git a/scripts/audit_harvester_hardware.py b/scripts/audit_harvester_hardware.py new file mode 100644 index 0000000..3b7e280 --- /dev/null +++ b/scripts/audit_harvester_hardware.py @@ -0,0 +1,75 @@ +import paramiko +import time +import re + +nodes = { + "Node 1 (snaihvs0lnx01)": "140.44.44.121", + "Node 2 (snaihvs0lnx02)": "140.44.44.122", + "Node 3 (snaihvs0lnx03)": "140.44.44.123", + "Node 4 (idrac-spacedock4)": "140.44.44.124", +} + +results = {} + +for name, ip in nodes.items(): + print(f"Connecting to {name} ({ip})...") + node_data = {} + try: + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username="root", password="Katzir476!", timeout=10) + chan = c.invoke_shell() + time.sleep(2) + if chan.recv_ready(): + chan.recv(4096) + + def send_and_recv(cmd, wait_time=2.5): + chan.send(cmd + "\r\n") + time.sleep(wait_time) + out = "" + while chan.recv_ready(): + out += chan.recv(8192).decode('utf-8', errors='ignore') + time.sleep(0.5) + return out + + # 1. BIOS Settings + raw_bios = send_and_recv("racadm get bios.SysProfileSettings", 2) + for key in ["SysProfile", "ProcCStates", "ProcC1E", "ProcTurboMode", "ProcPwrPerf"]: + m = re.search(rf"{key}\s*=\s*(\w+)", raw_bios) + node_data[key] = m.group(1) if m else "Unknown" + + raw_proc = send_and_recv("racadm get bios.ProcSettings", 2) + for key in ["ProcVirtualization", "ProcExecuteDisable", "LogicalProc"]: + m = re.search(rf"{key}\s*=\s*(\w+)", raw_proc) + node_data[key] = m.group(1) if m else "Unknown" + + raw_dev = send_and_recv("racadm get bios.IntegratedDevices", 2) + for key in ["SriovGlobalEnable"]: + m = re.search(rf"{key}\s*=\s*(\w+)", raw_dev) + node_data[key] = m.group(1) if m else "Unknown" + + # 2. Storage / RAID + raw_ctrl = send_and_recv("racadm raid get controllers", 2) + controllers = [line.strip() for line in raw_ctrl.splitlines() if "RAID." in line or "PERC" in line or "AHCI." in line] + node_data["Controllers"] = ", ".join(controllers) if controllers else "None" + + raw_vdisks = send_and_recv("racadm raid get vdisks", 3) + vdisks = [line.strip() for line in raw_vdisks.splitlines() if line.startswith("Disk.Virtual")] + node_data["VirtualDisks"] = len(vdisks) + node_data["VirtualDisksList"] = vdisks + + raw_pdisks = send_and_recv("racadm raid get pdisks", 4) + pdisk_lines = [line.strip() for line in raw_pdisks.splitlines() if line.startswith("Disk.Bay")] + node_data["PhysicalDisks"] = len(pdisk_lines) + node_data["PDisksSummary"] = raw_pdisks + + c.close() + except Exception as e: + node_data["Error"] = str(e) + + results[name] = node_data + +print("\n\n========================= SUMMARY MATRIX =========================") +import json +print(json.dumps(results, indent=2)) + diff --git a/scripts/audit_harvester_live_nodes.py b/scripts/audit_harvester_live_nodes.py new file mode 100644 index 0000000..2810498 --- /dev/null +++ b/scripts/audit_harvester_live_nodes.py @@ -0,0 +1,24 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', password='Katzir476!', timeout=10) + +def run(cmd): + stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'") + return stdout.read().decode().strip() + +print("=== CPU & VIRTUALIZATION (lscpu) ===") +print(run("lscpu | grep -E 'Model name|CPU\(s\):|Thread|Socket|Virtualization|NUMA'")) + +print("\n=== KERNEL PARAMETERS (/proc/cmdline) ===") +print(run("cat /proc/cmdline")) + +print("\n=== STORAGE CONTROLLER (lspci) ===") +print(run("lspci | grep -E 'RAID|SAS|SATA|Ethernet|Network'")) + +print("\n=== BLOCK DEVICES (lsblk) ===") +print(run("lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS")) + +c.close() + diff --git a/scripts/check_gpus_idrac.py b/scripts/check_gpus_idrac.py new file mode 100644 index 0000000..38533f0 --- /dev/null +++ b/scripts/check_gpus_idrac.py @@ -0,0 +1,37 @@ +import paramiko + +nodes = { + "Node 1 (140.44.44.121)": "140.44.44.121", + "Node 2 (140.44.44.122)": "140.44.44.122", + "Node 3 (140.44.44.123)": "140.44.44.123", + "Node 4 (140.44.44.124)": "140.44.44.124", +} + +for name, ip in nodes.items(): + try: + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username="root", password="Katzir476!", timeout=10) + + # Check PCIe inventory for NVIDIA + stdin, stdout, stderr = c.exec_command("racadm hwinventory") + out = stdout.read().decode(errors="ignore") + + gpus = [] + for block in out.split("[InstanceID:"): + if "NVIDIA" in block or "Tesla" in block or "T4" in block: + lines = [l.strip() for l in block.splitlines() if any(k in l for k in ["DeviceDescription", "DeviceType", "FQDD", "PCI"])] + gpus.append(" | ".join(lines[:3])) + + # Check Above 4G decoding + stdin, stdout, stderr = c.exec_command("racadm get bios.IntegratedDevices.MemOpMode || racadm get bios.IntegratedDevices.MemoryMappedIoAbove4GB") + a4g = stdout.read().decode(errors="ignore").strip() + + print(f"=== {name} ===") + print(f" GPUs detected: {len(gpus)}") + for g in gpus: + print(f" {g}") + c.close() + except Exception as e: + print(f"=== {name} === ERROR: {e}") + diff --git a/scripts/check_node4_specs.py b/scripts/check_node4_specs.py new file mode 100644 index 0000000..8cb86f2 --- /dev/null +++ b/scripts/check_node4_specs.py @@ -0,0 +1,28 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.44.124', username='root', password='Katzir476!', timeout=10) + +def query(cmd): + stdin, stdout, stderr = c.exec_command(f"racadm {cmd}") + return stdout.read().decode().strip() + +print("--- System Info ---") +print(query("getsysinfo")) + +print("\n--- BIOS SysProfile ---") +print(query("get bios.SysProfileSettings.SysProfile")) + +print("\n--- BIOS ProcVirtualization ---") +print(query("get bios.ProcSettings.ProcVirtualization")) + +print("\n--- BIOS SriovGlobalEnable ---") +print(query("get bios.IntegratedDevices.SriovGlobalEnable")) + +print("\n--- RAID Controllers ---") +print(query("raid get controllers")) + +c.close() + diff --git a/scripts/check_nodes2_3.py b/scripts/check_nodes2_3.py new file mode 100644 index 0000000..82354bf --- /dev/null +++ b/scripts/check_nodes2_3.py @@ -0,0 +1,22 @@ +import paramiko + +for ip, name in [("140.44.44.122", "Node 2"), ("140.44.44.123", "Node 3")]: + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', password='Katzir476!', timeout=10) + + stdin, stdout, stderr = c.exec_command('racadm get bios.SysProfileSettings.SysProfile') + prof = stdout.read().decode().strip().splitlines()[-1] + + c.close() + + # New connection for second command + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(ip, username='root', password='Katzir476!', timeout=10) + stdin, stdout, stderr = c.exec_command('racadm get bios.IntegratedDevices.SriovGlobalEnable') + sriov = stdout.read().decode().strip().splitlines()[-1] + c.close() + + print(f"[{name} - {ip}] {prof} | {sriov}") + diff --git a/scripts/check_redfish_node4.py b/scripts/check_redfish_node4.py new file mode 100644 index 0000000..810d89a --- /dev/null +++ b/scripts/check_redfish_node4.py @@ -0,0 +1,34 @@ +import urllib.request +import ssl +import json +import base64 + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +auth = base64.b64encode(b'root:Katzir476!').decode() +headers = {'Authorization': 'Basic ' + auth} + +# Query System +url = 'https://140.44.44.124/redfish/v1/Systems/System.Embedded.1' +req = urllib.request.Request(url, headers=headers) +with urllib.request.urlopen(req, context=ctx, timeout=10) as r: + sys_data = json.loads(r.read().decode()) + +print("Model:", sys_data.get("Model")) +print("BiosVersion:", sys_data.get("BiosVersion")) +print("ProcessorSummary:", sys_data.get("ProcessorSummary")) +print("MemorySummary:", sys_data.get("MemorySummary")) + +# Query BIOS attributes +bios_url = 'https://140.44.44.124/redfish/v1/Systems/System.Embedded.1/Bios' +req = urllib.request.Request(bios_url, headers=headers) +with urllib.request.urlopen(req, context=ctx, timeout=10) as r: + bios_data = json.loads(r.read().decode()) + attrs = bios_data.get("Attributes", {}) + print("\n=== BIOS KEY ATTRIBUTES ===") + for k in ["SysProfile", "ProcVirtualization", "SriovGlobalEnable", "ProcCStates", "ProcC1E", "LogicalProc", "MemOpMode"]: + print(f" {k}: {attrs.get(k)}") + + diff --git a/scripts/inspect_harvester_live.py b/scripts/inspect_harvester_live.py new file mode 100644 index 0000000..818c8c2 --- /dev/null +++ b/scripts/inspect_harvester_live.py @@ -0,0 +1,11 @@ +import paramiko + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.4.90', username='rancher', password='Katzir476!', timeout=10) + +cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; /var/lib/rancher/rke2/bin/kubectl get vms -A; /var/lib/rancher/rke2/bin/kubectl get volumes.longhorn.io -n longhorn-system | head -n 10'" +stdin, stdout, stderr = c.exec_command(cmd) +print("STDOUT:\n" + stdout.read().decode()) +print("STDERR:\n" + stderr.read().decode()) +c.close() diff --git a/scripts/post_switch_healthcheck.py b/scripts/post_switch_healthcheck.py new file mode 100644 index 0000000..f1ea2fb --- /dev/null +++ b/scripts/post_switch_healthcheck.py @@ -0,0 +1,20 @@ +import paramiko +import subprocess + +hosts = { + "Swarm Mgr (snshpd4lnxswm01)": "140.44.4.71", + "Aerodrome K3s (sninfr2lnxctl01)": "140.44.4.11", + "Spaceport K3s (sninfr0lnxctl01)": "140.44.4.41", + "Harvester VIP (spacedock)": "140.44.4.90", + "iDRAC Spacedock 1": "140.44.44.121", + "iDRAC Spacedock 2": "140.44.44.122", + "iDRAC Spacedock 3": "140.44.44.123", + "iDRAC Spacedock 4": "140.44.44.124", +} + +print("=== PING STATUS ===") +for name, ip in hosts.items(): + res = subprocess.run(["ping", "-n", "1", "-w", "1000", ip], capture_output=True) + status = "ONLINE" if res.returncode == 0 else "OFFLINE" + print(f"[{status:7s}] {name:32s} ({ip})") + diff --git a/scripts/test_racadm_queries.py b/scripts/test_racadm_queries.py new file mode 100644 index 0000000..9591315 --- /dev/null +++ b/scripts/test_racadm_queries.py @@ -0,0 +1,31 @@ +import paramiko +import time + +c = paramiko.SSHClient() +c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +c.connect('140.44.44.121', username='root', password='Katzir476!', timeout=10) + +chan = c.invoke_shell() +time.sleep(2) +if chan.recv_ready(): + chan.recv(4096) + +commands = [ + "racadm get bios.SysProfileSettings.SysProfile", + "racadm get bios.ProcSettings.ProcVirtualization", + "racadm get bios.IntegratedDevices.SriovGlobalEnable", + "racadm raid get controllers", + "racadm raid get pdisks" +] + +for cmd in commands: + chan.send(cmd + "\r\n") + time.sleep(2) + resp = "" + while chan.recv_ready(): + resp += chan.recv(4096).decode('utf-8', errors='ignore') + time.sleep(0.5) + print(f"=== {cmd} ===") + print(resp.strip()) + +c.close()