Compare commits
5 Commits
9391b3f2a1
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 8117326b14 | |||
| 4f0fa3fe55 | |||
| 42f669266e | |||
| 5a583d1db9 | |||
| 5cb00c04a9 |
@@ -1,10 +1,10 @@
|
||||
---
|
||||
- name: Aerodrome K3s Control Plane Sequential Minor Version Upgrade
|
||||
- name: Aerodrome K3s Control Plane Sequential Upgrade
|
||||
hosts: k3s_aerodrome_masters
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
vars:
|
||||
target_k3s_version: "v1.27.16+k3s1"
|
||||
target_k3s_version: "v1.31.4+k3s1"
|
||||
|
||||
tasks:
|
||||
- name: Query pre-upgrade K3s version
|
||||
@@ -16,7 +16,7 @@
|
||||
ansible.builtin.debug:
|
||||
msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
|
||||
|
||||
- name: Download target K3s release binary and replace executable
|
||||
- name: Download target K3s release binary
|
||||
ansible.builtin.shell: |
|
||||
set -e
|
||||
ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
|
||||
@@ -36,12 +36,12 @@
|
||||
ansible.builtin.command: systemctl is-active k3s
|
||||
register: k3s_active
|
||||
until: k3s_active.stdout == "active"
|
||||
retries: 20
|
||||
retries: 25
|
||||
delay: 4
|
||||
changed_when: false
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
|
||||
- name: Wait for upgraded node to report Ready in Kubernetes
|
||||
- name: Wait for upgraded master to report Ready in Kubernetes
|
||||
ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
|
||||
delegate_to: sninfr2lnxctl01
|
||||
register: node_ready
|
||||
@@ -63,7 +63,100 @@
|
||||
fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
|
||||
success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
|
||||
|
||||
- name: Settle etcd cluster state
|
||||
- name: Settle etcd cluster consensus
|
||||
ansible.builtin.pause:
|
||||
seconds: 10
|
||||
seconds: 15
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
|
||||
- name: Aerodrome K3s Worker Nodes Sequential Upgrade
|
||||
hosts: k3s_aerodrome_workers
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
vars:
|
||||
target_k3s_version: "v1.31.4+k3s1"
|
||||
|
||||
tasks:
|
||||
- name: Query pre-upgrade worker K3s version
|
||||
ansible.builtin.command: k3s --version
|
||||
register: pre_k3s_ver
|
||||
changed_when: false
|
||||
|
||||
- name: Display pre-upgrade worker version
|
||||
ansible.builtin.debug:
|
||||
msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
|
||||
|
||||
- name: Cordon worker node
|
||||
ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
|
||||
delegate_to: sninfr2lnxctl01
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
changed_when: true
|
||||
|
||||
- name: Drain worker node
|
||||
ansible.builtin.command: >
|
||||
kubectl drain {{ inventory_hostname }}
|
||||
--ignore-daemonsets
|
||||
--delete-emptydir-data
|
||||
--force
|
||||
--timeout=180s
|
||||
delegate_to: sninfr2lnxctl01
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
failed_when: false
|
||||
changed_when: true
|
||||
|
||||
- name: Download target K3s release binary on worker
|
||||
ansible.builtin.shell: |
|
||||
set -e
|
||||
ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
|
||||
curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
|
||||
chmod +x /usr/local/bin/k3s.new
|
||||
mv /usr/local/bin/k3s.new /usr/local/bin/k3s
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
register: k3s_download_worker
|
||||
|
||||
- name: Restart K3s Agent systemd service
|
||||
ansible.builtin.systemd:
|
||||
name: k3s-agent
|
||||
state: restarted
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
|
||||
- name: Wait for K3s Agent systemd unit to report active
|
||||
ansible.builtin.command: systemctl is-active k3s-agent
|
||||
register: agent_active
|
||||
until: agent_active.stdout == "active"
|
||||
retries: 25
|
||||
delay: 4
|
||||
changed_when: false
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
|
||||
- name: Uncordon worker node
|
||||
ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
|
||||
delegate_to: sninfr2lnxctl01
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for upgraded worker to report Ready in Kubernetes
|
||||
ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
|
||||
delegate_to: sninfr2lnxctl01
|
||||
register: node_ready
|
||||
until: node_ready.stdout == "True"
|
||||
retries: 30
|
||||
delay: 5
|
||||
changed_when: false
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
|
||||
- name: Query post-upgrade worker K3s version
|
||||
ansible.builtin.command: k3s --version
|
||||
register: post_k3s_ver
|
||||
changed_when: false
|
||||
|
||||
- name: Verify post-upgrade worker version matches target
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- target_k3s_version in post_k3s_ver.stdout
|
||||
fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
|
||||
success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
|
||||
|
||||
- name: Settle Longhorn storage replicas between node transitions
|
||||
ansible.builtin.pause:
|
||||
seconds: 15
|
||||
when: target_k3s_version not in pre_k3s_ver.stdout
|
||||
|
||||
@@ -5,10 +5,46 @@
|
||||
gather_facts: true
|
||||
|
||||
tasks:
|
||||
- name: Pre-seed debconf for non-interactive kernel and package upgrades
|
||||
ansible.builtin.shell: |
|
||||
echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
|
||||
echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
|
||||
when: ansible_os_family == "Debian"
|
||||
changed_when: false
|
||||
|
||||
- name: Ensure critical storage prerequisites are installed
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- open-iscsi
|
||||
- nfs-common
|
||||
state: present
|
||||
update_cache: false
|
||||
when: ansible_os_family == "Debian" and node_role is defined and node_role in ['master', 'worker']
|
||||
|
||||
- name: Cordon Kubernetes node if applicable
|
||||
ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
|
||||
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
|
||||
when: node_role is defined and node_role == 'worker'
|
||||
changed_when: true
|
||||
|
||||
- name: Drain Kubernetes node if applicable
|
||||
ansible.builtin.command: >
|
||||
kubectl drain {{ inventory_hostname }}
|
||||
--ignore-daemonsets
|
||||
--delete-emptydir-data
|
||||
--force
|
||||
--timeout=180s
|
||||
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
|
||||
when: node_role is defined and node_role == 'worker'
|
||||
failed_when: false
|
||||
changed_when: true
|
||||
|
||||
- name: Update apt cache
|
||||
ansible.builtin.apt:
|
||||
update_cache: true
|
||||
cache_valid_time: 3600
|
||||
environment:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
when: ansible_os_family == "Debian"
|
||||
|
||||
- name: Perform dist-upgrade
|
||||
@@ -16,6 +52,9 @@
|
||||
upgrade: dist
|
||||
autoremove: true
|
||||
autoclean: true
|
||||
dpkg_options: 'force-confdef,force-confold'
|
||||
environment:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
when: ansible_os_family == "Debian"
|
||||
|
||||
- name: Check if system reboot is required
|
||||
@@ -28,16 +67,18 @@
|
||||
block:
|
||||
- name: Reboot host gracefully
|
||||
ansible.builtin.reboot:
|
||||
msg: "Automated rolling maintenance reboot via AWX"
|
||||
msg: "Automated rolling maintenance reboot via Snarfnet Automation"
|
||||
reboot_timeout: 300
|
||||
connect_timeout: 30
|
||||
test_command: uptime
|
||||
|
||||
- name: Verify systemd-timesyncd is active after reboot
|
||||
ansible.builtin.systemd:
|
||||
name: systemd-timesyncd
|
||||
state: started
|
||||
ignore_errors: true
|
||||
- name: Verify systemd-timesyncd or chrony is active after reboot
|
||||
ansible.builtin.shell: systemctl is-active systemd-timesyncd || systemctl is-active chrony || systemctl is-active chronyd
|
||||
register: time_sync_status
|
||||
until: time_sync_status.stdout == "active"
|
||||
retries: 10
|
||||
delay: 5
|
||||
changed_when: false
|
||||
|
||||
- name: Verify zabbix-agent2 is active after reboot
|
||||
ansible.builtin.systemd:
|
||||
@@ -45,3 +86,25 @@
|
||||
state: started
|
||||
ignore_errors: true
|
||||
|
||||
- name: Uncordon Kubernetes node if applicable
|
||||
ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
|
||||
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
|
||||
when: node_role is defined and node_role == 'worker'
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for Kubernetes node to report Ready
|
||||
ansible.builtin.command: >
|
||||
kubectl get node {{ inventory_hostname }}
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
|
||||
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
|
||||
register: node_ready
|
||||
until: node_ready.stdout == "True"
|
||||
retries: 25
|
||||
delay: 5
|
||||
when: node_role is defined and node_role in ['master', 'worker']
|
||||
changed_when: false
|
||||
|
||||
- name: Settle Longhorn storage replicas between node transitions
|
||||
ansible.builtin.pause:
|
||||
seconds: 15
|
||||
when: node_role is defined and node_role in ['master', 'worker']
|
||||
|
||||
@@ -41,7 +41,7 @@ Our automated live probe across the homelab APIs identified the following baseli
|
||||
| **Aerodrome K3s (`local`)** | All 10 Nodes (`ctl01-03` & `wrk01-07`): **`v1.30.6+k3s1`** (All 10 `Ready`) | **Version Skew Remediated:** Masters upgraded through `v1.27.16` $\rightarrow$ `v1.28.15` $\rightarrow$ `v1.29.10` $\rightarrow$ `v1.30.6+k3s1` with zero workload downtime. Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active. Enrolled in ArgoCD. |
|
||||
| **Spaceport K3s (`c-m-4fgkl75b`)** | 4 Masters + 6 Workers: **`v1.31.4+k3s1`** (All 10 `Ready`) | Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active across Gitea, Zabbix Postgres, Jenkins, and Semaphore DBs. ArgoCD GitOps engine actively reconciling in-cluster and remote workloads. |
|
||||
| **Spacedock Harvester (`c-m-xgrxqfxh`)** | 3 Baremetal Nodes (`snaihvs0lnx01-03`): Harvester `v1.8.0` / `v1.35.2+rke2r1` | Healthy `harvester-longhorn` default StorageClass. Declarative VLAN networks and VM templates codified in Terraform. Monitored via SNMP / ICMP. |
|
||||
| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | NPM pinned in `PublishMode: host` on `.74` with daily NAS backup. Hardened NFS persistent mounts (`/mnt/nfs_shares`) on Synology NAS (`14.10.10.71:/volume1/docker`). Automated SQLite backup timer active with daily tarball sync to NAS. |
|
||||
| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | Dual-NPM HA Keepalived cluster active on VIP `140.44.4.70` across `snshpd4lnxswm01` (.71) and `snshpd4lnxswm02` (.72) in `PublishMode: host`. Hardened NFS persistent mounts (`/mnt/nfs_shares`) on Synology NAS (`14.10.10.71:/volume1/docker`). Automated SQLite backup timer active with daily tarball sync to NAS. |
|
||||
| **Zabbix (`zabbix.snarfnet.net`)** | Version `7.0.16` (`140.44.4.57:8888`, Trapper `140.44.4.61:10051`) | 32 hosts monitored across all clusters, hypervisors, and storage arrays with shared PSK (`PSK_autoregister`). Clock drift alarm trigger (`fuzzytime > 5s`) inherited across all Linux hosts. Gotify & Discord problem actions active. |
|
||||
| **Gotify (`gotify.snarfnet.net`)** | Version `2.9.1` (`140.44.4.70:8090`) | 5 structured notification channels provisioned. Internal VIP (`http://140.44.4.70:8090`) active for direct programmatic alerts. `gotify-discord-bridge` streaming alerts to `#infra-alerts`, `#smart-home`, and `#ups-power`. |
|
||||
| **AWX (`awx.snarfnet.net`)** | Operator `2.19.1` / AWX `24.6.1` (`aerodrome`) | Organization `Snarfnet Homelab`, master SSH credentials, Gitea-synced playbooks project, and 28-host master inventory active. Daily drift audit and rolling upgrade job templates verified green. |
|
||||
@@ -399,7 +399,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
|
||||
| Role | Hostname / Identifier | Target IP | Subnet / VLAN | Description & Failure Protocol |
|
||||
|---|---|---|---|---|
|
||||
| **NUT Master Server** | `nutserver.snarfnet.net` | `14.4.4.44` | VLAN 50 (Smart Home) | Primary NUT `upsd` daemon polling physical UPS via USB/Serial/SNMP. Runs `upsmon master` and holds `/root/.ssh/id_ed25519_snarfnet`. |
|
||||
| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.74` | VLAN 20 (Swarm 04) | NPM running in `PublishMode: host` on ports 80/443/81. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
|
||||
| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.70` | VLAN 20 (Keepalived VIP) | Dual-NPM active/standby running in `PublishMode: host` on ports 80/443/81 across Swarm 01 & 02. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
|
||||
| **Authentication & SSO** | `authentik.snarfnet.net` | `140.44.4.29:444` | VLAN 20 (Aerodrome K3s) | Handles Forward Auth & Conditional MFA. Runs on Longhorn storage in Aerodrome. |
|
||||
| **UPS Web Dashboard** | `peanut.snarfnet.net` | `140.44.4.70:8050` | VLAN 20 (Swarm VIP) | Peanut container UI consuming `14.4.4.44:3493`. Protected by Authentik Forward Auth externally. |
|
||||
| **Emergency Push & Discord** | `gotify.snarfnet.net` | `140.44.4.70:8090` | VLAN 20 (Swarm VIP) | Gotify server + `gotify-discord-bridge`. **Internal outage alerts use `http://140.44.4.70:8090`** (+ direct Discord webhook fallback) to bypass Authentik during shutdown. |
|
||||
@@ -466,7 +466,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
|
||||
3. **Stage 3 (NUT Master):** `14.4.4.44` boots (`BIOS: Restore on AC Power Loss = Always On`), starts `upsd`, and sends `ONLINE` alert to Discord/Gotify.
|
||||
4. **Stage 4 (Storage Gate — Must Complete Before Compute):** Synology NAS arrays (`14.10.10.71` & `14.10.10.72`) boot, mount Btrfs pools, and expose NFS (`:2049`).
|
||||
5. **Stage 5 (Control Planes & Hypervisors):** Harvester (`140.44.4.81-83`), K3s Masters (`aerodrome` & `spaceport`), and Swarm Managers (`140.44.4.71-75`) boot and establish etcd/Raft quorum.
|
||||
6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and NPM on `snshpd4lnxswm04` (`140.44.4.74`) binds host ports `80/443/81`.
|
||||
6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and Dual-NPM on Swarm 01 & 02 (`140.44.4.71-72`) binds host ports `80/443/81` behind Keepalived VIP `140.44.4.70`.
|
||||
|
||||
---
|
||||
|
||||
|
||||
30
scripts/check_db_creds.py
Normal file
30
scripts/check_db_creds.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
|
||||
|
||||
print("--- Testing authentik dump ---")
|
||||
cmd = "kubectl exec -n infra authentik-postgresql-0 -- env"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(stdout.read().decode())
|
||||
ssh.close()
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
|
||||
|
||||
print("--- Testing gitea env ---")
|
||||
cmd = "kubectl exec -n infrastructure gitea-postgresql-0 -- env"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("--- Testing zabbix env ---")
|
||||
cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- env"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(stdout.read().decode())
|
||||
ssh.close()
|
||||
|
||||
21
scripts/check_host_ports.py
Normal file
21
scripts/check_host_ports.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
for ip, node in [('140.44.4.71', 'snshpd4lnxswm01'), ('140.44.4.72', 'snshpd4lnxswm02')]:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print(f"=== {node} ({ip}) ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:81 |:443 " || true')
|
||||
print("SS:\n", stdout.read().decode())
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker ps -a --filter "name=nginx" --format "{{.ID}} {{.Status}} {{.Names}}"')
|
||||
print("Containers:\n", stdout.read().decode())
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
35
scripts/check_keepalived_all.py
Normal file
35
scripts/check_keepalived_all.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import paramiko
|
||||
|
||||
NODES = [
|
||||
("swm01", "140.44.4.71"),
|
||||
("swm02", "140.44.4.72"),
|
||||
("swm03", "140.44.4.73"),
|
||||
("swm04", "140.44.4.74"),
|
||||
("swm05", "140.44.4.75"),
|
||||
]
|
||||
|
||||
def main():
|
||||
for name, ip in NODES:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
|
||||
stdin, stdout, stderr = ssh.exec_command("systemctl is-active keepalived 2>/dev/null || echo inactive")
|
||||
status = stdout.read().decode().strip()
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command("ip -4 addr show | grep '140.44.4.70' || true")
|
||||
vip = stdout.read().decode().strip()
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command("grep -E 'state|priority' /etc/keepalived/keepalived.conf 2>/dev/null || true")
|
||||
conf = stdout.read().decode().strip()
|
||||
|
||||
print(f"[{name} - {ip}] Keepalived: {status} | VIP: {'YES' if '140.44.4.70' in vip else 'NO'}")
|
||||
if conf:
|
||||
print(f" Config: {conf.splitlines()}")
|
||||
ssh.close()
|
||||
except Exception as e:
|
||||
print(f"[{name} - {ip}] Error:", e)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
29
scripts/check_longhorn_settings.py
Normal file
29
scripts/check_longhorn_settings.py
Normal file
@@ -0,0 +1,29 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
for name, ip in [("aerodrome", "140.44.4.11"), ("spaceport", "140.44.4.41")]:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print(f"=== LONGHORN SETTINGS ON {name} ({ip}) ===")
|
||||
settings = [
|
||||
'auto-salvage-volume',
|
||||
'detach-manually-attached-volume-with-missing-node',
|
||||
'replica-replenishment-wait-interval',
|
||||
'pod-deletion-for-node-down-failure-handling',
|
||||
'backup-target'
|
||||
]
|
||||
for s in settings:
|
||||
cmd = f"sudo kubectl get setting.longhorn.io {s} -n longhorn-system -o jsonpath='{{.value}}' 2>/dev/null || echo 'not found'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
val = stdout.read().decode().strip()
|
||||
print(f" {s}: {val}")
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
14
scripts/check_nfs_mounts.py
Normal file
14
scripts/check_nfs_mounts.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
|
||||
cmd = "df -h | grep -i nfs || echo 'no nfs mounts'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(f"[{name.upper()}] NFS mounts:\n{stdout.read().decode().strip()}")
|
||||
ssh.close()
|
||||
|
||||
27
scripts/check_npm_status.py
Normal file
27
scripts/check_npm_status.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
for ip in ['140.44.4.71', '140.44.4.72']:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:443 |:81 " || true')
|
||||
print(f"=== LISTENING PORTS ON {ip} ===")
|
||||
print(stdout.read().decode())
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" --format "{{.ID}} {{.Status}}"')
|
||||
cid = stdout.read().decode().strip()
|
||||
print(f"Container on {ip}: {cid}")
|
||||
if cid:
|
||||
cid_clean = cid.split()[0]
|
||||
stdin, stdout, stderr = ssh.exec_command(f'sudo docker logs {cid_clean} --tail 15')
|
||||
print(f"Logs on {ip}:\n{stdout.read().decode()}")
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
16
scripts/check_npm_volumes.py
Normal file
16
scripts/check_npm_volumes.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
for ip in ['140.44.4.71', '140.44.4.72']:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
stdin, stdout, stderr = ssh.exec_command("sudo docker volume inspect nginx-proxy-manager_nginx-data --format '{{json .Options}}'")
|
||||
print(f"Volume data on {ip}: {stdout.read().decode().strip()}")
|
||||
stdin, stdout, stderr = ssh.exec_command("sudo docker volume inspect nginx-proxy-manager_nginx-letsencrypt --format '{{json .Options}}'")
|
||||
print(f"Volume letsencrypt on {ip}: {stdout.read().decode().strip()}")
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
16
scripts/check_port_forwards.py
Normal file
16
scripts/check_port_forwards.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
ssh.connect('14.44.4.1', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
|
||||
stdin, stdout, stderr = ssh.exec_command('iptables -t nat -S | grep -E "80|443"')
|
||||
print("IPTABLES NAT:\n", stdout.read().decode())
|
||||
ssh.close()
|
||||
except Exception as e:
|
||||
print("SSH to UDM Pro error:", e)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
15
scripts/check_swarm_ports.py
Normal file
15
scripts/check_swarm_ports.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
for ip in ['140.44.4.71', '140.44.4.72']:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo ss -tulpn | grep -E ":80 |:443 |:81 " || true')
|
||||
print(f"=== PORTS ON {ip} ===")
|
||||
print(stdout.read().decode())
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
23
scripts/check_swarm_service_status.py
Normal file
23
scripts/check_swarm_service_status.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== DOCKER SERVICE LS ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker service ls')
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("=== DOCKER SERVICE PS ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app')
|
||||
print(stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
60
scripts/deploy_dual_npm.py
Normal file
60
scripts/deploy_dual_npm.py
Normal file
@@ -0,0 +1,60 @@
|
||||
import paramiko
|
||||
import time
|
||||
import urllib.request
|
||||
import ssl
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== STEP 1: Updating nginx-proxy-manager_app to Dual-Node Swarm Placement ===")
|
||||
cmd = (
|
||||
"sudo docker service update "
|
||||
"--constraint-rm 'node.hostname == snshpd4lnxswm04' "
|
||||
"--constraint-add 'node.labels.npm == true' "
|
||||
"--replicas 2 "
|
||||
"--update-order start-first "
|
||||
"nginx-proxy-manager_app"
|
||||
)
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print("Service update output:\n", stdout.read().decode())
|
||||
print("Service update stderr:\n", stderr.read().decode())
|
||||
|
||||
print("=== STEP 2: Waiting for tasks to stabilize ===")
|
||||
time.sleep(10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command("sudo docker service ps nginx-proxy-manager_app --no-trunc")
|
||||
print("Tasks:\n", stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
print("=== STEP 3: Verifying HTTP / HTTPS endpoints ===")
|
||||
for ip in ['140.44.4.70', '140.44.4.71', '140.44.4.72']:
|
||||
# Port 80
|
||||
try:
|
||||
req = urllib.request.Request(f"http://{ip}", headers={"Host": "test.snarfnet.net"})
|
||||
with urllib.request.urlopen(req, timeout=4) as r:
|
||||
print(f"[{ip}:80] HTTP {r.status}")
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f"[{ip}:80] HTTP {e.code} (NPM responding)")
|
||||
except Exception as e:
|
||||
print(f"[{ip}:80] Error: {e}")
|
||||
|
||||
# Port 81 (Admin UI)
|
||||
try:
|
||||
req = urllib.request.Request(f"http://{ip}:81")
|
||||
with urllib.request.urlopen(req, timeout=4) as r:
|
||||
print(f"[{ip}:81] HTTP {r.status} (Admin UI responding)")
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f"[{ip}:81] HTTP {e.code}")
|
||||
except Exception as e:
|
||||
print(f"[{ip}:81] Error: {e}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
65
scripts/dump_k8s_databases.py
Normal file
65
scripts/dump_k8s_databases.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import datetime
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
db_targets = [
|
||||
{
|
||||
"cluster": "aerodrome",
|
||||
"ip": "140.44.4.11",
|
||||
"namespace": "infra",
|
||||
"pod": "authentik-postgresql-0",
|
||||
"name": "authentik",
|
||||
"dump_cmd": "kubectl exec -n infra authentik-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/authentik_db_{timestamp}.sql.gz"
|
||||
},
|
||||
{
|
||||
"cluster": "aerodrome",
|
||||
"ip": "140.44.4.11",
|
||||
"namespace": "awx",
|
||||
"pod": "awx-postgres-15-0",
|
||||
"name": "awx",
|
||||
"dump_cmd": "kubectl exec -n awx awx-postgres-15-0 -- pg_dumpall -U postgres | gzip > /var/backups/awx_db_{timestamp}.sql.gz"
|
||||
},
|
||||
{
|
||||
"cluster": "spaceport",
|
||||
"ip": "140.44.4.41",
|
||||
"namespace": "infrastructure",
|
||||
"pod": "gitea-postgresql-0",
|
||||
"name": "gitea",
|
||||
"dump_cmd": "kubectl exec -n infrastructure gitea-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/gitea_db_{timestamp}.sql.gz"
|
||||
},
|
||||
{
|
||||
"cluster": "spaceport",
|
||||
"ip": "140.44.4.41",
|
||||
"namespace": "monitoring",
|
||||
"pod": "zabbix-postgresql-0",
|
||||
"name": "zabbix",
|
||||
"dump_cmd": "kubectl exec -n monitoring zabbix-postgresql-0 -- pg_dumpall -U postgres | gzip > /var/backups/zabbix_db_{timestamp}.sql.gz"
|
||||
}
|
||||
]
|
||||
|
||||
for target in db_targets:
|
||||
cluster = target["cluster"]
|
||||
ip = target["ip"]
|
||||
name = target["name"]
|
||||
print(f"[{cluster.upper()}] Backing up {name} database...")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=15)
|
||||
|
||||
cmd = f"mkdir -p /var/backups && {target['dump_cmd'].replace('{timestamp}', timestamp)}"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=300)
|
||||
exit_status = stdout.channel.recv_exit_status()
|
||||
err = stderr.read().decode().strip()
|
||||
|
||||
if exit_status == 0:
|
||||
check_cmd = f"ls -lh /var/backups/{name}_db_{timestamp}.sql.gz"
|
||||
_, chk_out, _ = ssh.exec_command(check_cmd)
|
||||
print(f" SUCCESS: {chk_out.read().decode().strip()}")
|
||||
else:
|
||||
print(f" FAILED (exit {exit_status}): {err}")
|
||||
|
||||
ssh.close()
|
||||
25
scripts/inspect_container_nginx.py
Normal file
25
scripts/inspect_container_nginx.py
Normal file
@@ -0,0 +1,25 @@
|
||||
import paramiko
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" -q')
|
||||
cid = stdout.read().decode().strip()
|
||||
print("Container ID:", cid)
|
||||
|
||||
if cid:
|
||||
stdin, stdout, stderr = ssh.exec_command(f'sudo docker exec {cid} nginx -t')
|
||||
print("NGINX -t STDOUT:\n", stdout.read().decode())
|
||||
print("NGINX -t STDERR:\n", stderr.read().decode())
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command(f'sudo docker exec {cid} curl -I http://127.0.0.1:80')
|
||||
print("CURL 80 INSIDE:\n", stdout.read().decode())
|
||||
print("CURL 80 STDERR:\n", stderr.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
29
scripts/inspect_db_env.py
Normal file
29
scripts/inspect_db_env.py
Normal file
@@ -0,0 +1,29 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
|
||||
|
||||
print("=== AERODROME DB USERS/ENV ===")
|
||||
for ns, pod in [('infra', 'authentik-postgresql-0'), ('awx', 'awx-postgres-15-0')]:
|
||||
cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
|
||||
|
||||
ssh.close()
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
|
||||
|
||||
print("=== SPACEPORT DB USERS/ENV ===")
|
||||
for ns, pod in [('infrastructure', 'gitea-postgresql-0'), ('monitoring', 'zabbix-postgresql-0')]:
|
||||
cmd = f"kubectl get pod {pod} -n {ns} -o jsonpath='{{.spec.containers[0].env}}'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(f"[{ns}/{pod}] Env: {stdout.read().decode()[:200]}")
|
||||
|
||||
ssh.close()
|
||||
|
||||
19
scripts/inspect_db_pods.py
Normal file
19
scripts/inspect_db_pods.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
for name, ip in [('aerodrome', '140.44.4.11'), ('spaceport', '140.44.4.41')]:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=8)
|
||||
cmd = "kubectl get pods -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
lines = stdout.read().decode().splitlines()
|
||||
print(f"=== {name.upper()} PODS ===")
|
||||
for line in lines:
|
||||
lower = line.lower()
|
||||
if any(k in lower for k in ['postgres', 'mysql', 'authentik', 'gitea', 'zabbix', 'awx', 'semaphore']):
|
||||
print(" " + line)
|
||||
ssh.close()
|
||||
|
||||
26
scripts/inspect_keepalived_conf.py
Normal file
26
scripts/inspect_keepalived_conf.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== /etc/keepalived/keepalived.conf ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo cat /etc/keepalived/keepalived.conf')
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("=== /etc/keepalived/scripts/ ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo ls -la /etc/keepalived/scripts/ || true')
|
||||
print(stdout.read().decode())
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo cat /etc/keepalived/scripts/* || true')
|
||||
print(stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
34
scripts/inspect_npm_72.py
Normal file
34
scripts/inspect_npm_72.py
Normal file
@@ -0,0 +1,34 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== DOCKER EXEC PS INSIDE f026d1c24ef6 ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker exec f026d1c24ef6 ps aux')
|
||||
print(stdout.read().decode())
|
||||
print(stderr.read().decode())
|
||||
|
||||
print("=== REDIRECTION_HOST LS ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo ls -la /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/redirection_host || true')
|
||||
print(stdout.read().decode())
|
||||
print(stderr.read().decode())
|
||||
|
||||
print("=== RECENT DOCKER LOGS WITH TIMESTAMPS ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker logs f026d1c24ef6 --tail 30')
|
||||
print(stdout.read().decode())
|
||||
|
||||
print("=== LOCAL CURL TEST ON 140.44.4.72 ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('curl -I http://127.0.0.1:80 ; curl -I http://127.0.0.1:81')
|
||||
print(stdout.read().decode())
|
||||
print(stderr.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
25
scripts/inspect_swarm_nodes_labels.py
Normal file
25
scripts/inspect_swarm_nodes_labels.py
Normal file
@@ -0,0 +1,25 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker node ls --format "{{.Hostname}}"')
|
||||
nodes = stdout.read().decode().split()
|
||||
for n in nodes:
|
||||
stdin, stdout, stderr = ssh.exec_command(f'sudo docker node inspect {n} --format "{{{{.Description.Hostname}}}}: {{{{.Spec.Labels}}}}"')
|
||||
print(stdout.read().decode().strip())
|
||||
|
||||
print("\n=== DOCKER SERVICE PS ALL ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app --no-trunc')
|
||||
print(stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
21
scripts/list_longhorn_settings.py
Normal file
21
scripts/list_longhorn_settings.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
stdin, stdout, stderr = ssh.exec_command('kubectl get settings.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,VALUE:.value')
|
||||
lines = stdout.read().decode().splitlines()
|
||||
for l in lines:
|
||||
for kw in ['salvage', 'detach', 'replenish', 'pod', 'node-down']:
|
||||
if kw in l.lower():
|
||||
print(l)
|
||||
break
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -51,3 +51,4 @@ def main():
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
||||
30
scripts/rescale_npm.py
Normal file
30
scripts/rescale_npm.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import sys
|
||||
import time
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== SCALING TO 1 ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker service scale nginx-proxy-manager_app=1')
|
||||
print(stdout.read().decode())
|
||||
time.sleep(3)
|
||||
|
||||
print("=== SCALING TO 2 ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker service scale nginx-proxy-manager_app=2')
|
||||
print(stdout.read().decode())
|
||||
time.sleep(5)
|
||||
|
||||
print("=== SERVICE PS ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker service ps nginx-proxy-manager_app --filter desired-state=running')
|
||||
print(stdout.read().decode())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
24
scripts/setup_dual_npm_cluster.py
Normal file
24
scripts/setup_dual_npm_cluster.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
def main():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== STEP 1: Add node labels 'npm=true' to swm01 (.71) and swm02 (.72) ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker node update --label-add npm=true snshpd4lnxswm01 && sudo docker node update --label-add npm=true snshpd4lnxswm02')
|
||||
print("Labels update:", stdout.read().decode().strip(), stderr.read().decode().strip())
|
||||
|
||||
# Check node labels
|
||||
stdin, stdout, stderr = ssh.exec_command("sudo docker node inspect snshpd4lnxswm01 --format '{{json .Spec.Labels}}'")
|
||||
print("Labels swm01:", stdout.read().decode().strip())
|
||||
stdin, stdout, stderr = ssh.exec_command("sudo docker node inspect snshpd4lnxswm02 --format '{{json .Spec.Labels}}'")
|
||||
print("Labels swm02:", stdout.read().decode().strip())
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
37
scripts/sync_backups_to_nas.py
Normal file
37
scripts/sync_backups_to_nas.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
nodes = [
|
||||
("aerodrome", "140.44.4.11", ["authentik_clean.sql.gz", "awx_clean.sql.gz"]),
|
||||
("spaceport", "140.44.4.41", ["gitea_clean.sql.gz", "zabbix_clean.sql.gz"])
|
||||
]
|
||||
|
||||
for name, ip, files in nodes:
|
||||
print(f"[{name.upper()}] Mounting NAS and copying backups...")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
setup_cmd = """
|
||||
mkdir -p /mnt/nas_backup
|
||||
mount -t nfs -o vers=4 14.10.10.71:/volume1/docker /mnt/nas_backup || mount -t nfs 14.10.10.71:/volume1/docker /mnt/nas_backup
|
||||
mkdir -p /mnt/nas_backup/backups/databases
|
||||
"""
|
||||
stdin, stdout, stderr = ssh.exec_command(setup_cmd)
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
for f in files:
|
||||
cp_cmd = f"cp /var/backups/{f} /mnt/nas_backup/backups/databases/ && ls -lh /mnt/nas_backup/backups/databases/{f}"
|
||||
stdin, stdout, stderr = ssh.exec_command(cp_cmd)
|
||||
stdout.channel.recv_exit_status()
|
||||
print(f" Saved to NAS: {stdout.read().decode().strip()}")
|
||||
|
||||
umount_cmd = "umount -f /mnt/nas_backup || true"
|
||||
stdin, stdout, stderr = ssh.exec_command(umount_cmd)
|
||||
stdout.channel.recv_exit_status()
|
||||
ssh.close()
|
||||
|
||||
print("All database backups verified mirrored to Synology NAS (14.10.10.71).")
|
||||
|
||||
15
scripts/test_authentik_dump.py
Normal file
15
scripts/test_authentik_dump.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U authentik authentik' | gzip > /var/backups/authentik_clean.sql.gz"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print("STDERR:", stderr.read().decode())
|
||||
_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
|
||||
print("RESULT:", out.read().decode().strip())
|
||||
ssh.close()
|
||||
|
||||
40
scripts/test_clean_db_dumps.py
Normal file
40
scripts/test_clean_db_dumps.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
# Authentik
|
||||
print("=== Backing up Authentik ===")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
cmd = "kubectl exec -n infra authentik-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/postgres-password) pg_dump -U postgres authentik' | gzip > /var/backups/authentik_clean.sql.gz"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
err = stderr.read().decode().strip()
|
||||
if err: print("Authentik err:", err)
|
||||
_, out, _ = ssh.exec_command("ls -lh /var/backups/authentik_clean.sql.gz")
|
||||
print("Authentik size:", out.read().decode().strip())
|
||||
ssh.close()
|
||||
|
||||
# Spaceport (Gitea & Zabbix)
|
||||
print("=== Backing up Gitea & Zabbix ===")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
cmd = "mkdir -p /var/backups && kubectl exec -n infrastructure gitea-postgresql-0 -- sh -c 'PGPASSWORD=$(cat /opt/bitnami/postgresql/secrets/password) pg_dump -U ugandanese baptiste' | gzip > /var/backups/gitea_clean.sql.gz"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
err = stderr.read().decode().strip()
|
||||
if err: print("Gitea err:", err)
|
||||
_, out, _ = ssh.exec_command("ls -lh /var/backups/gitea_clean.sql.gz")
|
||||
print("Gitea size:", out.read().decode().strip())
|
||||
|
||||
cmd = "kubectl exec -n monitoring zabbix-postgresql-0 -- sh -c 'PGPASSWORD=zabbix pg_dump -U zabbix zabbix' | gzip > /var/backups/zabbix_clean.sql.gz"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
err = stderr.read().decode().strip()
|
||||
if err: print("Zabbix err:", err)
|
||||
_, out, _ = ssh.exec_command("ls -lh /var/backups/zabbix_clean.sql.gz")
|
||||
print("Zabbix size:", out.read().decode().strip())
|
||||
|
||||
ssh.close()
|
||||
|
||||
35
scripts/test_etcd_snapshots.py
Normal file
35
scripts/test_etcd_snapshots.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import datetime
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
clusters = [
|
||||
("aerodrome", "140.44.4.11"),
|
||||
("spaceport", "140.44.4.41")
|
||||
]
|
||||
|
||||
for name, ip in clusters:
|
||||
print(f"[{name.upper()}] Connecting to {ip}...")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
cmd = f"k3s etcd-snapshot save --name pre-upgrade-{name}-{timestamp}"
|
||||
print(f"[{name.upper()}] Executing: {cmd}")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
if out:
|
||||
print(f"[{name.upper()}] STDOUT: {out}")
|
||||
if err:
|
||||
print(f"[{name.upper()}] STDERR: {err}")
|
||||
|
||||
check_cmd = "k3s etcd-snapshot ls"
|
||||
stdin, stdout, stderr = ssh.exec_command(check_cmd)
|
||||
print(f"[{name.upper()}] Recent Snapshots:\n" + stdout.read().decode().strip())
|
||||
|
||||
ssh.close()
|
||||
print("-" * 50)
|
||||
67
scripts/test_keepalived_failover.py
Normal file
67
scripts/test_keepalived_failover.py
Normal file
@@ -0,0 +1,67 @@
|
||||
import sys
|
||||
import time
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def check_vip(ssh):
|
||||
stdin, stdout, stderr = ssh.exec_command("ip -4 addr show dev eth0 | grep '140.44.4.70' || true")
|
||||
return bool(stdout.read().decode().strip())
|
||||
|
||||
def curl_vip(ssh):
|
||||
stdin, stdout, stderr = ssh.exec_command("curl -s -o /dev/null -w '%{http_code}' -m 2 http://140.44.4.70:80 || true")
|
||||
return stdout.read().decode().strip()
|
||||
|
||||
def main():
|
||||
ssh1 = paramiko.SSHClient()
|
||||
ssh1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh1.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
ssh2 = paramiko.SSHClient()
|
||||
ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh2.connect('140.44.4.72', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print("=== INITIAL STATE ===")
|
||||
v1 = check_vip(ssh1)
|
||||
v2 = check_vip(ssh2)
|
||||
code = curl_vip(ssh1)
|
||||
print(f"swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
|
||||
|
||||
print("\n=== TRIGGERING FAILOVER: STOPPING KEEPALIVED ON swm01 ===")
|
||||
ssh1.exec_command("sudo systemctl stop keepalived")
|
||||
|
||||
# Wait up to 5 seconds for swm02 to claim VIP
|
||||
claimed = False
|
||||
for i in range(5):
|
||||
time.sleep(1)
|
||||
if check_vip(ssh2):
|
||||
claimed = True
|
||||
break
|
||||
|
||||
v1 = check_vip(ssh1)
|
||||
v2 = check_vip(ssh2)
|
||||
code = curl_vip(ssh2)
|
||||
print(f"Failover result -> swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
|
||||
|
||||
print("\n=== RESTORING swm01: STARTING KEEPALIVED ON swm01 ===")
|
||||
ssh1.exec_command("sudo systemctl start keepalived")
|
||||
|
||||
# Wait up to 5 seconds for swm01 to reclaim VIP
|
||||
reclaimed = False
|
||||
for i in range(5):
|
||||
time.sleep(1)
|
||||
if check_vip(ssh1):
|
||||
reclaimed = True
|
||||
break
|
||||
|
||||
v1 = check_vip(ssh1)
|
||||
v2 = check_vip(ssh2)
|
||||
code = curl_vip(ssh1)
|
||||
print(f"Fallback result -> swm01 has VIP: {v1}, swm02 has VIP: {v2}, HTTP code on VIP: {code}")
|
||||
|
||||
ssh1.close()
|
||||
ssh2.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
26
scripts/test_npm_both_nodes.py
Normal file
26
scripts/test_npm_both_nodes.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
for ip, node in [('140.44.4.71', 'snshpd4lnxswm01'), ('140.44.4.72', 'snshpd4lnxswm02')]:
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
print(f"=== TESTING {node} ({ip}) ===")
|
||||
stdin, stdout, stderr = ssh.exec_command('sudo docker ps --filter "name=nginx-proxy-manager_app" --format "{{.ID}} {{.Status}} {{.Names}}"')
|
||||
print("Containers:", stdout.read().decode().strip())
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command('curl -I http://127.0.0.1:80 ; curl -I http://127.0.0.1:81')
|
||||
print("Curls:\n", stdout.read().decode())
|
||||
err = stderr.read().decode()
|
||||
if err:
|
||||
print("Errors:\n", err)
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
113
scripts/test_worker_os_patch.py
Normal file
113
scripts/test_worker_os_patch.py
Normal file
@@ -0,0 +1,113 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
MASTER_IP = '140.44.4.11'
|
||||
WORKER_IP = '140.44.4.27'
|
||||
NODE_NAME = 'sninfr2lnxwrk07'
|
||||
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
|
||||
|
||||
print(f"=== STEP 1: TEST ROLLING OS PATCH ON {NODE_NAME} ({WORKER_IP}) ===")
|
||||
|
||||
# 1. Connect to master to check pods on node
|
||||
master_ssh = paramiko.SSHClient()
|
||||
master_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
master_ssh.connect(MASTER_IP, username='root', key_filename=KEY_FILE, timeout=10)
|
||||
|
||||
print(f"[*] Checking active pods on {NODE_NAME}...")
|
||||
_, stdout, _ = master_ssh.exec_command(f"kubectl get pods -A --field-selector spec.nodeName={NODE_NAME}")
|
||||
print(stdout.read().decode())
|
||||
|
||||
print(f"[*] Cordoning {NODE_NAME}...")
|
||||
_, stdout, stderr = master_ssh.exec_command(f"kubectl cordon {NODE_NAME}")
|
||||
print(" " + stdout.read().decode().strip())
|
||||
|
||||
print(f"[*] Draining {NODE_NAME}...")
|
||||
_, stdout, stderr = master_ssh.exec_command(f"kubectl drain {NODE_NAME} --ignore-daemonsets --delete-emptydir-data --force --timeout=180s")
|
||||
print(" " + stdout.read().decode().strip())
|
||||
err = stderr.read().decode().strip()
|
||||
if err:
|
||||
print(" (drain notice):", err)
|
||||
|
||||
# 2. Connect to worker to perform apt-get update & dist-upgrade
|
||||
print(f"[*] Connecting directly to worker {WORKER_IP} for package upgrades...")
|
||||
worker_ssh = paramiko.SSHClient()
|
||||
worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=10)
|
||||
|
||||
upgrade_cmd = "DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade -y && apt-get autoremove -y"
|
||||
print("[*] Running dist-upgrade (preserving configurations)...")
|
||||
stdin, stdout, stderr = worker_ssh.exec_command(upgrade_cmd)
|
||||
exit_status = stdout.channel.recv_exit_status()
|
||||
print(f"[*] dist-upgrade completed with exit code: {exit_status}")
|
||||
|
||||
# 3. Check reboot requirement
|
||||
_, stdout, _ = worker_ssh.exec_command("test -f /var/run/reboot-required && echo 'yes' || echo 'no'")
|
||||
reboot_needed = stdout.read().decode().strip() == 'yes'
|
||||
print(f"[*] Reboot required: {reboot_needed}")
|
||||
|
||||
if reboot_needed:
|
||||
print("[*] Rebooting worker node...")
|
||||
try:
|
||||
worker_ssh.exec_command("reboot")
|
||||
except Exception:
|
||||
pass
|
||||
worker_ssh.close()
|
||||
|
||||
print("[*] Waiting 30 seconds for host to reboot...")
|
||||
time.sleep(30)
|
||||
|
||||
# Poll SSH until online
|
||||
reconnected = False
|
||||
for attempt in range(25):
|
||||
try:
|
||||
worker_ssh = paramiko.SSHClient()
|
||||
worker_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
worker_ssh.connect(WORKER_IP, username='root', key_filename=KEY_FILE, timeout=6)
|
||||
reconnected = True
|
||||
print(f"[*] Worker reconnected successfully on attempt {attempt+1}!")
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(5)
|
||||
|
||||
if not reconnected:
|
||||
print("[!] ERROR: Worker node did not come back online within timeout!")
|
||||
sys.exit(1)
|
||||
|
||||
# Verify services
|
||||
_, stdout, _ = worker_ssh.exec_command("systemctl is-active systemd-timesyncd || systemctl is-active chrony")
|
||||
print(f" TimeSync: {stdout.read().decode().strip()}")
|
||||
_, stdout, _ = worker_ssh.exec_command("systemctl is-active zabbix-agent2")
|
||||
print(f" ZabbixAgent2: {stdout.read().decode().strip()}")
|
||||
_, stdout, _ = worker_ssh.exec_command("systemctl is-active k3s-agent || systemctl is-active k3s")
|
||||
print(f" K3sAgent: {stdout.read().decode().strip()}")
|
||||
worker_ssh.close()
|
||||
else:
|
||||
worker_ssh.close()
|
||||
|
||||
# 4. Uncordon node
|
||||
print(f"[*] Uncordoning {NODE_NAME}...")
|
||||
_, stdout, _ = master_ssh.exec_command(f"kubectl uncordon {NODE_NAME}")
|
||||
print(" " + stdout.read().decode().strip())
|
||||
|
||||
# 5. Wait for node to be Ready
|
||||
print(f"[*] Verifying node status in Kubernetes...")
|
||||
for _ in range(15):
|
||||
_, stdout, _ = master_ssh.exec_command(f"kubectl get node {NODE_NAME} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}'")
|
||||
status = stdout.read().decode().strip()
|
||||
if status == "True":
|
||||
print(f"[OK] Node {NODE_NAME} is fully Ready!")
|
||||
break
|
||||
time.sleep(4)
|
||||
|
||||
# 6. Check Longhorn volume robustness
|
||||
print("[*] Settling Longhorn volume replicas...")
|
||||
time.sleep(10)
|
||||
_, stdout, _ = master_ssh.exec_command("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness")
|
||||
print(stdout.read().decode())
|
||||
|
||||
master_ssh.close()
|
||||
print(f"[SUCCESS] Step 1 Test Worker OS Patch on {NODE_NAME} completed flawlessly!")
|
||||
|
||||
34
scripts/tune_longhorn_settings.py
Normal file
34
scripts/tune_longhorn_settings.py
Normal file
@@ -0,0 +1,34 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def main():
|
||||
clusters = [
|
||||
("aerodrome", "140.44.4.11"),
|
||||
("spaceport", "140.44.4.41")
|
||||
]
|
||||
for name, host in clusters:
|
||||
print(f"=== Configuring Longhorn settings on {name} ({host}) ===")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(host, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
# 1. replica-replenishment-wait-interval -> 300
|
||||
cmd1 = "kubectl patch setting.longhorn.io replica-replenishment-wait-interval -n longhorn-system --type=merge -p '{\"value\":\"300\"}'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd1)
|
||||
print("replica-replenishment-wait-interval:", stdout.read().decode().strip(), stderr.read().decode().strip())
|
||||
|
||||
# 2. node-down-pod-deletion-policy -> delete-deployment-pod
|
||||
cmd2 = "kubectl patch setting.longhorn.io node-down-pod-deletion-policy -n longhorn-system --type=merge -p '{\"value\":\"delete-deployment-pod\"}'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd2)
|
||||
print("node-down-pod-deletion-policy:", stdout.read().decode().strip(), stderr.read().decode().strip())
|
||||
|
||||
# Verify
|
||||
stdin, stdout, stderr = ssh.exec_command("kubectl get setting.longhorn.io replica-replenishment-wait-interval node-down-pod-deletion-policy -n longhorn-system -o custom-columns=NAME:.metadata.name,VALUE:.value")
|
||||
print("VERIFIED:\n", stdout.read().decode().strip())
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
50
scripts/update_keepalived_healthcheck.py
Normal file
50
scripts/update_keepalived_healthcheck.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import sys
|
||||
import paramiko
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
NODES = [
|
||||
("swm01", "140.44.4.71"),
|
||||
("swm02", "140.44.4.72"),
|
||||
("swm03", "140.44.4.73"),
|
||||
("swm04", "140.44.4.74"),
|
||||
("swm05", "140.44.4.75"),
|
||||
]
|
||||
|
||||
SCRIPT_CONTENT = """#!/bin/bash
|
||||
# Check docker daemon health
|
||||
systemctl is-active --quiet docker || exit 1
|
||||
|
||||
# If NPM container is running on this node, verify HTTP responsiveness
|
||||
if docker ps --format '{{.Names}}' 2>/dev/null | grep -q "nginx-proxy-manager_app"; then
|
||||
curl -sf -m 2 http://127.0.0.1:80 >/dev/null 2>&1 || curl -sf -m 2 http://127.0.0.1:81 >/dev/null 2>&1 || exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
"""
|
||||
|
||||
def main():
|
||||
for name, ip in NODES:
|
||||
print(f"=== Updating {name} ({ip}) ===")
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
# Write to temp file then sudo mv
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/check_docker.sh', 'w') as f:
|
||||
f.write(SCRIPT_CONTENT.replace('\r\n', '\n'))
|
||||
sftp.close()
|
||||
|
||||
cmd = "sudo mv /tmp/check_docker.sh /etc/keepalived/scripts/check_docker.sh && sudo chmod +x /etc/keepalived/scripts/check_docker.sh && sudo /etc/keepalived/scripts/check_docker.sh && echo 'TEST: OK'"
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
print(stdout.read().decode().strip())
|
||||
err = stderr.read().decode().strip()
|
||||
if err:
|
||||
print("ERR:", err)
|
||||
|
||||
ssh.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user