feat: add various infrastructure check and automation scripts
This commit is contained in:
146
scripts/deploy_mount_units_all_nodes.py
Normal file
146
scripts/deploy_mount_units_all_nodes.py
Normal file
@@ -0,0 +1,146 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
|
||||
nodes = {
|
||||
"snaisvr44hrvstr01": "140.44.4.81",
|
||||
"snaisvr44hrvstr02": "140.44.4.82",
|
||||
"snaisvr44hrvstr03": "140.44.4.83",
|
||||
"snaisvr44hrvstr04": "140.44.4.84"
|
||||
}
|
||||
|
||||
# Python script that runs on each node
|
||||
node_script = """python3 - << 'PYEOF'
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import glob
|
||||
|
||||
# Clean up any bad units first
|
||||
for f in glob.glob("/etc/systemd/system/var-lib-harvester-extra*"):
|
||||
if os.path.isfile(f) or os.path.islink(f):
|
||||
os.remove(f)
|
||||
|
||||
# 1. Map physical disks to their Longhorn diskUUID
|
||||
out = subprocess.check_output("blkid", shell=True).decode()
|
||||
disks = {}
|
||||
for line in out.splitlines():
|
||||
if not line.startswith("/dev/sd"): continue
|
||||
dev = line.split(":")[0]
|
||||
if "sda" in dev: continue
|
||||
uuid = ""
|
||||
for part in line.split():
|
||||
if part.startswith("UUID="):
|
||||
uuid = part.split('"')[1]
|
||||
if uuid:
|
||||
disks[dev] = uuid
|
||||
|
||||
disk_info = {} # diskUUID -> fs_uuid
|
||||
for dev, uuid in disks.items():
|
||||
tmp = f"/tmp/m_{os.path.basename(dev)}"
|
||||
os.makedirs(tmp, exist_ok=True)
|
||||
try:
|
||||
subprocess.check_call(f"mount {dev} {tmp}", shell=True)
|
||||
cfg_file = os.path.join(tmp, "longhorn-disk.cfg")
|
||||
if os.path.exists(cfg_file):
|
||||
with open(cfg_file) as f:
|
||||
data = json.load(f)
|
||||
lh_uuid = data.get("diskUUID")
|
||||
disk_info[lh_uuid] = uuid
|
||||
subprocess.call(f"umount {tmp}", shell=True)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 2. Read expected from /tmp/node_expected.json
|
||||
with open("/tmp/node_expected.json") as f:
|
||||
expected = json.load(f)
|
||||
|
||||
created_units = []
|
||||
for disk_name, info in expected.items():
|
||||
lh_uuid = info["diskUUID"]
|
||||
path = info["path"]
|
||||
if not path or path == "/var/lib/harvester/defaultdisk":
|
||||
continue
|
||||
|
||||
if lh_uuid in disk_info:
|
||||
fs_uuid = disk_info[lh_uuid]
|
||||
os.makedirs(path, exist_ok=True)
|
||||
# Use systemd-escape binary to get the exact canonical unit filename
|
||||
unit_name = subprocess.check_output(["systemd-escape", "-p", "--suffix=mount", path]).decode().strip()
|
||||
unit_path = os.path.join("/etc/systemd/system", unit_name)
|
||||
|
||||
unit_content = f\"\"\"[Unit]
|
||||
Description=Harvester Extra Disk {disk_name}
|
||||
Before=rke2-server.service rke2-agent.service
|
||||
|
||||
[Mount]
|
||||
What=/dev/disk/by-uuid/{fs_uuid}
|
||||
Where={path}
|
||||
Type=ext4
|
||||
Options=defaults
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
\"\"\"
|
||||
with open(unit_path, "w") as uf:
|
||||
uf.write(unit_content)
|
||||
|
||||
created_units.append(unit_name)
|
||||
# Ensure it is mounted right now
|
||||
mounts = subprocess.check_output("mount", shell=True).decode()
|
||||
if path not in mounts:
|
||||
subprocess.call(f"mount /dev/disk/by-uuid/{fs_uuid} {path}", shell=True)
|
||||
|
||||
subprocess.call(["systemctl", "daemon-reload"])
|
||||
for u in created_units:
|
||||
subprocess.call(["systemctl", "enable", u])
|
||||
subprocess.call(["systemctl", "start", u])
|
||||
|
||||
print("Configured & mounted units:", created_units)
|
||||
PYEOF
|
||||
"""
|
||||
|
||||
# Fetch latest expected disks from Harvester VIP
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
_, o, _ = c.exec_command("echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; kubectl get nodes.longhorn.io -n longhorn-system -o json'")
|
||||
data = json.loads(o.read().decode())
|
||||
c.close()
|
||||
|
||||
all_expected = {}
|
||||
for node in data.get("items", []):
|
||||
nname = node["metadata"]["name"]
|
||||
all_expected[nname] = {}
|
||||
for dname, dstat in node.get("status", {}).get("diskStatus", {}).items():
|
||||
all_expected[nname][dname] = {
|
||||
"path": node.get("spec", {}).get("disks", {}).get(dname, {}).get("path"),
|
||||
"diskUUID": dstat.get("diskUUID")
|
||||
}
|
||||
|
||||
for hname, ip in nodes.items():
|
||||
print(f"\n=================== Deploying to {hname} ({ip}) ===================")
|
||||
expected_for_node = all_expected.get(hname, {})
|
||||
nc = paramiko.SSHClient()
|
||||
nc.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
nc.connect(ip, username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
# Write /tmp/node_expected.json via stdin
|
||||
stdin, stdout, stderr = nc.exec_command("cat > /tmp/node_expected.json")
|
||||
stdin.write(json.dumps(expected_for_node))
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
_, o, e = nc.exec_command(f"echo Katzir476! | sudo -S {node_script}")
|
||||
print(o.read().decode())
|
||||
if e.read().decode():
|
||||
pass
|
||||
|
||||
_, o, _ = nc.exec_command("df -h | grep extra-disks")
|
||||
print(f"Active mounts on {hname}:\n" + o.read().decode())
|
||||
nc.close()
|
||||
except Exception as e:
|
||||
print(f"Error on {hname}: {e}")
|
||||
|
||||
print("\nAll nodes deployed successfully!")
|
||||
Reference in New Issue
Block a user