feat: add DNS setup blueprint and infrastructure automation scripts

docs/DNS_Setup_Plan.md

@@ -180,3 +180,5 @@ We will define firewall groups on the UDM Pro:
   * Secondary: `140.44.4.65`
 * Set UDM Pro WAN DNS to `140.44.4.35` & `140.44.4.65` (or Quad9 fallback).
 * Apply LAN IN firewall rule matrix on UDM Pro to enforce strict VLAN isolation.
+
+snaisvr44hrvstr0x - AppropriateBeav3r39!

scripts/apply_bios_tuning.py

@@ -0,0 +1,51 @@
+import paramiko
+import time
+
+targets = [
+    {
+        "name": "Node 1 (140.44.44.121)",
+        "ip": "140.44.44.121",
+        "settings": [
+            ("bios.IntegratedDevices.SriovGlobalEnable", "Enabled")
+        ]
+    },
+    {
+        "name": "Node 4 (140.44.44.124)",
+        "ip": "140.44.44.124",
+        "settings": [
+            ("bios.SysProfileSettings.SysProfile", "PerfOptimized")
+        ]
+    }
+]
+
+for t in targets:
+    print(f"\n==========================================")
+    print(f"Applying BIOS settings to {t['name']}...")
+    try:
+        c = paramiko.SSHClient()
+        c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        c.connect(t['ip'], username='root', password='Katzir476!', timeout=15)
+        
+        for key, val in t['settings']:
+            cmd = f"racadm set {key} {val}"
+            stdin, stdout, stderr = c.exec_command(cmd)
+            out = stdout.read().decode().strip()
+            print(f"  racadm set {key} {val}:\n    {out}")
+            time.sleep(1)
+
+        # Create BIOS job
+        stdin, stdout, stderr = c.exec_command("racadm jobqueue create BIOS.Setup.1-1")
+        job_out = stdout.read().decode().strip()
+        print(f"  racadm jobqueue create BIOS.Setup.1-1:\n    {job_out}")
+        
+        # View job queue
+        stdin, stdout, stderr = c.exec_command("racadm jobqueue view")
+        q_out = stdout.read().decode().strip()
+        print(f"  Job Queue:\n    {q_out[:200]}")
+        
+        c.close()
+    except Exception as e:
+        print(f"  Error on {t['name']}: {e}")
+
+print("\nDone applying BIOS settings!")
+

scripts/audit_harvester_hardware.py

@@ -0,0 +1,75 @@
+import paramiko
+import time
+import re
+
+nodes = {
+    "Node 1 (snaihvs0lnx01)": "140.44.44.121",
+    "Node 2 (snaihvs0lnx02)": "140.44.44.122",
+    "Node 3 (snaihvs0lnx03)": "140.44.44.123",
+    "Node 4 (idrac-spacedock4)": "140.44.44.124",
+}
+
+results = {}
+
+for name, ip in nodes.items():
+    print(f"Connecting to {name} ({ip})...")
+    node_data = {}
+    try:
+        c = paramiko.SSHClient()
+        c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        c.connect(ip, username="root", password="Katzir476!", timeout=10)
+        chan = c.invoke_shell()
+        time.sleep(2)
+        if chan.recv_ready():
+            chan.recv(4096)
+            
+        def send_and_recv(cmd, wait_time=2.5):
+            chan.send(cmd + "\r\n")
+            time.sleep(wait_time)
+            out = ""
+            while chan.recv_ready():
+                out += chan.recv(8192).decode('utf-8', errors='ignore')
+                time.sleep(0.5)
+            return out
+
+        # 1. BIOS Settings
+        raw_bios = send_and_recv("racadm get bios.SysProfileSettings", 2)
+        for key in ["SysProfile", "ProcCStates", "ProcC1E", "ProcTurboMode", "ProcPwrPerf"]:
+            m = re.search(rf"{key}\s*=\s*(\w+)", raw_bios)
+            node_data[key] = m.group(1) if m else "Unknown"
+
+        raw_proc = send_and_recv("racadm get bios.ProcSettings", 2)
+        for key in ["ProcVirtualization", "ProcExecuteDisable", "LogicalProc"]:
+            m = re.search(rf"{key}\s*=\s*(\w+)", raw_proc)
+            node_data[key] = m.group(1) if m else "Unknown"
+
+        raw_dev = send_and_recv("racadm get bios.IntegratedDevices", 2)
+        for key in ["SriovGlobalEnable"]:
+            m = re.search(rf"{key}\s*=\s*(\w+)", raw_dev)
+            node_data[key] = m.group(1) if m else "Unknown"
+
+        # 2. Storage / RAID
+        raw_ctrl = send_and_recv("racadm raid get controllers", 2)
+        controllers = [line.strip() for line in raw_ctrl.splitlines() if "RAID." in line or "PERC" in line or "AHCI." in line]
+        node_data["Controllers"] = ", ".join(controllers) if controllers else "None"
+
+        raw_vdisks = send_and_recv("racadm raid get vdisks", 3)
+        vdisks = [line.strip() for line in raw_vdisks.splitlines() if line.startswith("Disk.Virtual")]
+        node_data["VirtualDisks"] = len(vdisks)
+        node_data["VirtualDisksList"] = vdisks
+
+        raw_pdisks = send_and_recv("racadm raid get pdisks", 4)
+        pdisk_lines = [line.strip() for line in raw_pdisks.splitlines() if line.startswith("Disk.Bay")]
+        node_data["PhysicalDisks"] = len(pdisk_lines)
+        node_data["PDisksSummary"] = raw_pdisks
+
+        c.close()
+    except Exception as e:
+        node_data["Error"] = str(e)
+
+    results[name] = node_data
+
+print("\n\n========================= SUMMARY MATRIX =========================")
+import json
+print(json.dumps(results, indent=2))
+

scripts/audit_harvester_live_nodes.py

@@ -0,0 +1,24 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', password='Katzir476!', timeout=10)
+
+def run(cmd):
+    stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
+    return stdout.read().decode().strip()
+
+print("=== CPU & VIRTUALIZATION (lscpu) ===")
+print(run("lscpu | grep -E 'Model name|CPU\(s\):|Thread|Socket|Virtualization|NUMA'"))
+
+print("\n=== KERNEL PARAMETERS (/proc/cmdline) ===")
+print(run("cat /proc/cmdline"))
+
+print("\n=== STORAGE CONTROLLER (lspci) ===")
+print(run("lspci | grep -E 'RAID|SAS|SATA|Ethernet|Network'"))
+
+print("\n=== BLOCK DEVICES (lsblk) ===")
+print(run("lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS"))
+
+c.close()
+

scripts/check_gpus_idrac.py

@@ -0,0 +1,37 @@
+import paramiko
+
+nodes = {
+    "Node 1 (140.44.44.121)": "140.44.44.121",
+    "Node 2 (140.44.44.122)": "140.44.44.122",
+    "Node 3 (140.44.44.123)": "140.44.44.123",
+    "Node 4 (140.44.44.124)": "140.44.44.124",
+}
+
+for name, ip in nodes.items():
+    try:
+        c = paramiko.SSHClient()
+        c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        c.connect(ip, username="root", password="Katzir476!", timeout=10)
+        
+        # Check PCIe inventory for NVIDIA
+        stdin, stdout, stderr = c.exec_command("racadm hwinventory")
+        out = stdout.read().decode(errors="ignore")
+        
+        gpus = []
+        for block in out.split("[InstanceID:"):
+            if "NVIDIA" in block or "Tesla" in block or "T4" in block:
+                lines = [l.strip() for l in block.splitlines() if any(k in l for k in ["DeviceDescription", "DeviceType", "FQDD", "PCI"])]
+                gpus.append(" | ".join(lines[:3]))
+        
+        # Check Above 4G decoding
+        stdin, stdout, stderr = c.exec_command("racadm get bios.IntegratedDevices.MemOpMode || racadm get bios.IntegratedDevices.MemoryMappedIoAbove4GB")
+        a4g = stdout.read().decode(errors="ignore").strip()
+
+        print(f"=== {name} ===")
+        print(f"  GPUs detected: {len(gpus)}")
+        for g in gpus:
+            print(f"    {g}")
+        c.close()
+    except Exception as e:
+        print(f"=== {name} === ERROR: {e}")
+

scripts/check_node4_specs.py

@@ -0,0 +1,28 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.44.124', username='root', password='Katzir476!', timeout=10)
+
+def query(cmd):
+    stdin, stdout, stderr = c.exec_command(f"racadm {cmd}")
+    return stdout.read().decode().strip()
+
+print("--- System Info ---")
+print(query("getsysinfo"))
+
+print("\n--- BIOS SysProfile ---")
+print(query("get bios.SysProfileSettings.SysProfile"))
+
+print("\n--- BIOS ProcVirtualization ---")
+print(query("get bios.ProcSettings.ProcVirtualization"))
+
+print("\n--- BIOS SriovGlobalEnable ---")
+print(query("get bios.IntegratedDevices.SriovGlobalEnable"))
+
+print("\n--- RAID Controllers ---")
+print(query("raid get controllers"))
+
+c.close()
+

scripts/check_nodes2_3.py

@@ -0,0 +1,22 @@
+import paramiko
+
+for ip, name in [("140.44.44.122", "Node 2"), ("140.44.44.123", "Node 3")]:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(ip, username='root', password='Katzir476!', timeout=10)
+    
+    stdin, stdout, stderr = c.exec_command('racadm get bios.SysProfileSettings.SysProfile')
+    prof = stdout.read().decode().strip().splitlines()[-1]
+    
+    c.close()
+    
+    # New connection for second command
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(ip, username='root', password='Katzir476!', timeout=10)
+    stdin, stdout, stderr = c.exec_command('racadm get bios.IntegratedDevices.SriovGlobalEnable')
+    sriov = stdout.read().decode().strip().splitlines()[-1]
+    c.close()
+
+    print(f"[{name} - {ip}] {prof} | {sriov}")
+

scripts/check_redfish_node4.py

@@ -0,0 +1,34 @@
+import urllib.request
+import ssl
+import json
+import base64
+
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+
+auth = base64.b64encode(b'root:Katzir476!').decode()
+headers = {'Authorization': 'Basic ' + auth}
+
+# Query System
+url = 'https://140.44.44.124/redfish/v1/Systems/System.Embedded.1'
+req = urllib.request.Request(url, headers=headers)
+with urllib.request.urlopen(req, context=ctx, timeout=10) as r:
+    sys_data = json.loads(r.read().decode())
+
+print("Model:", sys_data.get("Model"))
+print("BiosVersion:", sys_data.get("BiosVersion"))
+print("ProcessorSummary:", sys_data.get("ProcessorSummary"))
+print("MemorySummary:", sys_data.get("MemorySummary"))
+
+# Query BIOS attributes
+bios_url = 'https://140.44.44.124/redfish/v1/Systems/System.Embedded.1/Bios'
+req = urllib.request.Request(bios_url, headers=headers)
+with urllib.request.urlopen(req, context=ctx, timeout=10) as r:
+    bios_data = json.loads(r.read().decode())
+    attrs = bios_data.get("Attributes", {})
+    print("\n=== BIOS KEY ATTRIBUTES ===")
+    for k in ["SysProfile", "ProcVirtualization", "SriovGlobalEnable", "ProcCStates", "ProcC1E", "LogicalProc", "MemOpMode"]:
+        print(f"  {k}: {attrs.get(k)}")
+
+

scripts/inspect_harvester_live.py

@@ -0,0 +1,11 @@
+import paramiko
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.4.90', username='rancher', password='Katzir476!', timeout=10)
+
+cmd = "echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; /var/lib/rancher/rke2/bin/kubectl get vms -A; /var/lib/rancher/rke2/bin/kubectl get volumes.longhorn.io -n longhorn-system | head -n 10'"
+stdin, stdout, stderr = c.exec_command(cmd)
+print("STDOUT:\n" + stdout.read().decode())
+print("STDERR:\n" + stderr.read().decode())
+c.close()

scripts/post_switch_healthcheck.py

@@ -0,0 +1,20 @@
+import paramiko
+import subprocess
+
+hosts = {
+    "Swarm Mgr (snshpd4lnxswm01)": "140.44.4.71",
+    "Aerodrome K3s (sninfr2lnxctl01)": "140.44.4.11",
+    "Spaceport K3s (sninfr0lnxctl01)": "140.44.4.41",
+    "Harvester VIP (spacedock)": "140.44.4.90",
+    "iDRAC Spacedock 1": "140.44.44.121",
+    "iDRAC Spacedock 2": "140.44.44.122",
+    "iDRAC Spacedock 3": "140.44.44.123",
+    "iDRAC Spacedock 4": "140.44.44.124",
+}
+
+print("=== PING STATUS ===")
+for name, ip in hosts.items():
+    res = subprocess.run(["ping", "-n", "1", "-w", "1000", ip], capture_output=True)
+    status = "ONLINE" if res.returncode == 0 else "OFFLINE"
+    print(f"[{status:7s}] {name:32s} ({ip})")
+

scripts/test_racadm_queries.py

@@ -0,0 +1,31 @@
+import paramiko
+import time
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('140.44.44.121', username='root', password='Katzir476!', timeout=10)
+
+chan = c.invoke_shell()
+time.sleep(2)
+if chan.recv_ready():
+    chan.recv(4096)
+
+commands = [
+    "racadm get bios.SysProfileSettings.SysProfile",
+    "racadm get bios.ProcSettings.ProcVirtualization",
+    "racadm get bios.IntegratedDevices.SriovGlobalEnable",
+    "racadm raid get controllers",
+    "racadm raid get pdisks"
+]
+
+for cmd in commands:
+    chan.send(cmd + "\r\n")
+    time.sleep(2)
+    resp = ""
+    while chan.recv_ready():
+        resp += chan.recv(4096).decode('utf-8', errors='ignore')
+        time.sleep(0.5)
+    print(f"=== {cmd} ===")
+    print(resp.strip())
+
+c.close()