import paramiko

nodes = {
    "Node 1 (140.44.44.121)": "140.44.44.121",
    "Node 2 (140.44.44.122)": "140.44.44.122",
    "Node 3 (140.44.44.123)": "140.44.44.123",
    "Node 4 (140.44.44.124)": "140.44.44.124",
}

for name, ip in nodes.items():
    try:
        c = paramiko.SSHClient()
        c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        c.connect(ip, username="root", password="Katzir476!", timeout=10)
        
        # Check PCIe inventory for NVIDIA
        stdin, stdout, stderr = c.exec_command("racadm hwinventory")
        out = stdout.read().decode(errors="ignore")
        
        gpus = []
        for block in out.split("[InstanceID:"):
            if "NVIDIA" in block or "Tesla" in block or "T4" in block:
                lines = [l.strip() for l in block.splitlines() if any(k in l for k in ["DeviceDescription", "DeviceType", "FQDD", "PCI"])]
                gpus.append(" | ".join(lines[:3]))
        
        # Check Above 4G decoding
        stdin, stdout, stderr = c.exec_command("racadm get bios.IntegratedDevices.MemOpMode || racadm get bios.IntegratedDevices.MemoryMappedIoAbove4GB")
        a4g = stdout.read().decode(errors="ignore").strip()

        print(f"=== {name} ===")
        print(f"  GPUs detected: {len(gpus)}")
        for g in gpus:
            print(f"    {g}")
        c.close()
    except Exception as e:
        print(f"=== {name} === ERROR: {e}")