From 5a583d1db98a9bbcc55525d0a021223a0c0c75e8 Mon Sep 17 00:00:00 2001
From: Snarf <snarf@snarfnet.net>
Date: Tue, 3 Nov 2026 10:50:37 -0500
Subject: [PATCH] feat(pillar-3): tune longhorn settings for power resilience
 on aerodrome and spaceport

---
 scripts/check_longhorn_settings.py | 29 ++++++++++++++++++++++++++
 scripts/list_longhorn_settings.py  | 21 +++++++++++++++++++
 scripts/tune_longhorn_settings.py  | 33 ++++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+)
 create mode 100644 scripts/check_longhorn_settings.py
 create mode 100644 scripts/list_longhorn_settings.py
 create mode 100644 scripts/tune_longhorn_settings.py

diff --git a/scripts/check_longhorn_settings.py b/scripts/check_longhorn_settings.py
new file mode 100644
index 0000000..a4f95e7
--- /dev/null
+++ b/scripts/check_longhorn_settings.py
@@ -0,0 +1,29 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    for name, ip in [("aerodrome", "140.44.4.11"), ("spaceport", "140.44.4.41")]:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+        
+        print(f"=== LONGHORN SETTINGS ON {name} ({ip}) ===")
+        settings = [
+            'auto-salvage-volume',
+            'detach-manually-attached-volume-with-missing-node',
+            'replica-replenishment-wait-interval',
+            'pod-deletion-for-node-down-failure-handling',
+            'backup-target'
+        ]
+        for s in settings:
+            cmd = f"sudo kubectl get setting.longhorn.io {s} -n longhorn-system -o jsonpath='{{.value}}' 2>/dev/null || echo 'not found'"
+            stdin, stdout, stderr = ssh.exec_command(cmd)
+            val = stdout.read().decode().strip()
+            print(f"  {s}: {val}")
+            
+        ssh.close()
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/list_longhorn_settings.py b/scripts/list_longhorn_settings.py
new file mode 100644
index 0000000..3842712
--- /dev/null
+++ b/scripts/list_longhorn_settings.py
@@ -0,0 +1,21 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+    stdin, stdout, stderr = ssh.exec_command('kubectl get settings.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,VALUE:.value')
+    lines = stdout.read().decode().splitlines()
+    for l in lines:
+        for kw in ['salvage', 'detach', 'replenish', 'pod', 'node-down']:
+            if kw in l.lower():
+                print(l)
+                break
+    ssh.close()
+
+if __name__ == '__main__':
+    main()
+
diff --git a/scripts/tune_longhorn_settings.py b/scripts/tune_longhorn_settings.py
new file mode 100644
index 0000000..21a75cf
--- /dev/null
+++ b/scripts/tune_longhorn_settings.py
@@ -0,0 +1,33 @@
+import sys
+import paramiko
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+def main():
+    clusters = [
+        ("aerodrome", "140.44.4.11"),
+        ("spaceport", "140.44.4.41")
+    ]
+    for name, host in clusters:
+        print(f"=== Configuring Longhorn settings on {name} ({host}) ===")
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(host, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+        # 1. replica-replenishment-wait-interval -> 300
+        cmd1 = "kubectl patch setting.longhorn.io replica-replenishment-wait-interval -n longhorn-system --type=merge -p '{\"value\":\"300\"}'"
+        stdin, stdout, stderr = ssh.exec_command(cmd1)
+        print("replica-replenishment-wait-interval:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+        # 2. node-down-pod-deletion-policy -> delete-deployment-pod
+        cmd2 = "kubectl patch setting.longhorn.io node-down-pod-deletion-policy -n longhorn-system --type=merge -p '{\"value\":\"delete-deployment-pod\"}'"
+        stdin, stdout, stderr = ssh.exec_command(cmd2)
+        print("node-down-pod-deletion-policy:", stdout.read().decode().strip(), stderr.read().decode().strip())
+
+        # Verify
+        stdin, stdout, stderr = ssh.exec_command("kubectl get setting.longhorn.io replica-replenishment-wait-interval node-down-pod-deletion-policy -n longhorn-system -o custom-columns=NAME:.metadata.name,VALUE:.value")
+        print("VERIFIED:\n", stdout.read().decode().strip())
+        ssh.close()
+
+if __name__ == '__main__':
+    main()