feat: add various infrastructure check and automation scripts

This commit is contained in:
2026-11-15 17:17:41 -05:00
parent 336b7759d0
commit b301aa3ece
232 changed files with 8274 additions and 298 deletions

View File

@@ -0,0 +1,141 @@
import paramiko
import json
import time
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
nodes = {
"snaisvr44hrvstr01": "140.44.4.81",
"snaisvr44hrvstr02": "140.44.4.82",
"snaisvr44hrvstr03": "140.44.4.83",
"snaisvr44hrvstr04": "140.44.4.84"
}
# Python script to run on each node
remote_mount_script = """python3 - << 'PYEOF'
import os
import subprocess
import json
# 1. Map physical disks to their Longhorn diskUUID
out = subprocess.check_output("blkid", shell=True).decode()
disks = {}
for line in out.splitlines():
if not line.startswith("/dev/sd"): continue
dev = line.split(":")[0]
if "sda" in dev: continue
uuid = ""
for part in line.split():
if part.startswith("UUID="):
uuid = part.split('"')[1]
if uuid:
disks[dev] = uuid
disk_info = {} # diskUUID -> fs_uuid
for dev, uuid in disks.items():
tmp = f"/tmp/m_{os.path.basename(dev)}"
os.makedirs(tmp, exist_ok=True)
try:
subprocess.check_call(f"mount {dev} {tmp}", shell=True)
cfg_file = os.path.join(tmp, "longhorn-disk.cfg")
if os.path.exists(cfg_file):
with open(cfg_file) as f:
data = json.load(f)
lh_uuid = data.get("diskUUID")
disk_info[lh_uuid] = uuid
subprocess.call(f"umount {tmp}", shell=True)
except Exception as e:
pass
print("Disks probed with valid longhorn-disk.cfg:", len(disk_info))
# 2. Get Longhorn expected disk mapping from env or file
# We read expected disk mapping from /tmp/node_expected.json
with open("/tmp/node_expected.json") as f:
expected = json.load(f)
for disk_name, info in expected.items():
lh_uuid = info["diskUUID"]
path = info["path"]
if not path or path == "/var/lib/harvester/defaultdisk":
continue
if lh_uuid in disk_info:
fs_uuid = disk_info[lh_uuid]
print(f"Mounting {disk_name} ({lh_uuid}) with fs UUID {fs_uuid} -> {path}")
os.makedirs(path, exist_ok=True)
# Check if already mounted
mounts = subprocess.check_output("mount", shell=True).decode()
if path not in mounts:
subprocess.call(f"mount /dev/disk/by-uuid/{fs_uuid} {path}", shell=True)
# Write systemd mount unit
unit_name = f"var-lib-harvester-extrax2ddisks-{disk_name.replace('disk-', '')}.mount"
unit_content = f\"\"\"[Unit]
Description=Harvester Extra Disk {disk_name}
Before=rke2-server.service rke2-agent.service
[Mount]
What=/dev/disk/by-uuid/{fs_uuid}
Where={path}
Type=ext4
Options=defaults
[Install]
WantedBy=multi-user.target
\"\"\"
with open(f"/etc/systemd/system/{unit_name}", "w") as uf:
uf.write(unit_content)
print(f"Created /etc/systemd/system/{unit_name}")
else:
print(f"WARNING: Disk {disk_name} ({lh_uuid}) not found on this node!")
subprocess.call("systemctl daemon-reload", shell=True)
PYEOF
"""
# Read node expected from earlier dump
with open("scripts/dump_lh_disk_uuids.py") as f:
pass
# We will fetch latest expected directly
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
_, o, _ = c.exec_command("echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; kubectl get nodes.longhorn.io -n longhorn-system -o json'")
data = json.loads(o.read().decode())
c.close()
all_expected = {}
for node in data.get("items", []):
nname = node["metadata"]["name"]
all_expected[nname] = {}
for dname, dstat in node.get("status", {}).get("diskStatus", {}).items():
all_expected[nname][dname] = {
"path": node.get("spec", {}).get("disks", {}).get(dname, {}).get("path"),
"diskUUID": dstat.get("diskUUID")
}
for hname, ip in nodes.items():
print(f"\n=================== Configuring {hname} ({ip}) ===================")
expected_for_node = all_expected.get(hname, {})
nc = paramiko.SSHClient()
nc.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
nc.connect(ip, username="rancher", key_filename=SSH_KEY, timeout=10)
# Write expected json
exp_str = json.dumps(expected_for_node)
nc.exec_command(f"cat << 'EOF' > /tmp/node_expected.json\n{exp_str}\nEOF")
# Run remote mount script
_, o, e = nc.exec_command(f"echo Katzir476! | sudo -S {remote_mount_script}")
print(o.read().decode())
print(e.read().decode())
# Check df -h on node
_, o, _ = nc.exec_command("df -h | grep extra-disks")
print("Mounted extra-disks on", hname, ":\n", o.read().decode())
nc.close()
except Exception as e:
print("Error on", hname, ":", e)
print("\nDone configuring all nodes!")