Compare commits

...

65 Commits

Author SHA1 Message Date
b301aa3ece feat: add various infrastructure check and automation scripts 2026-11-15 17:17:41 -05:00
336b7759d0 feat: add infrastructure automation and diagnostic scripts for Zabbix, Nginx, and Authentik 2026-11-12 20:18:37 -05:00
68c108cfe8 docs(topology): regenerate high-res PNG diagram to reflect latest layout, hardware and storage metrics 2026-11-12 17:13:51 -05:00
9ebd48af0c chore: update homelab-gitops-repo submodule reference 2026-11-12 17:09:51 -05:00
113ebdc7c5 feat: add automation and inspection scripts for Nginx Proxy Manager and Authentik integration 2026-11-12 17:08:14 -05:00
5a70460e89 feat: add homelab deployment configs, documentation, and automation scripts 2026-11-12 16:57:08 -05:00
1228b957de chore: remove empty file 2026-11-08 15:03:27 -05:00
3c373351b7 fix: remove empty file diff 2026-11-08 14:57:08 -05:00
e4962b7334 chore: remove unused file 2026-11-08 14:17:31 -05:00
7f07fa5a89 chore: remove empty file 2026-11-08 13:51:41 -05:00
fc42bd6f94 feat: add DNS setup blueprint and infrastructure automation scripts 2026-11-08 09:28:06 -05:00
17e60354a1 feat(idrac): add iDRAC cluster check script and update redirection helpers 2026-11-07 21:09:41 -05:00
1d4dfb594b feat(dns): point snarfhome.snarfnet.net to 14.4.4.20 and remove Nabu Casa redirection 2026-11-07 18:21:46 -05:00
42bfb43b65 feat: add scripts to inspect snarfhome proxy and redirection configurations via SSH 2026-11-07 16:56:11 -05:00
bbcf656a3d feat: add automation scripts for inspecting proxy providers and updating Nginx configurations 2026-11-07 15:41:04 -05:00
e0ebc120ee Configure VaultConfigurator direct proxy in NPM and update topology docs 2026-11-07 15:19:19 -05:00
40e504c2fe feat(vault): add vault.snarfnet.net proxy host and topology docs 2026-11-07 14:41:42 -05:00
34cfece0b0 feat: add diagnostic and maintenance scripts for Pi-hole, Technitium, and Nginx Proxy Manager 2026-11-07 13:43:11 -05:00
4e8f93173e docs(topology): update AI stack hostnames to snnpud4lnxops01/snnpud4lnxweb01 and prune obsolete shipyardcontrol 2026-11-07 13:37:27 -05:00
1fe16429a4 feat: add script to check pihole-FTL database integrity and query count via SSH 2026-11-07 13:02:12 -05:00
ac1395486d feat: add script to apply DNS manifests via SSH to cluster nodes 2026-11-07 12:44:53 -05:00
a07cecb6f3 feat: add script to apply DNS manifests via SSH and kubectl 2026-11-07 12:42:44 -05:00
ffc9ab8ab0 feat: add diagnostic and traffic policy management scripts for DNS and Pi-hole 2026-11-07 12:42:31 -05:00
d4b367794c feat: add OpenCanary honeypot documentation and management scripts 2026-11-07 00:32:19 -05:00
3871d04d48 feat(security): deploy OpenCanary honeypot on VLAN 50 with Gotify and Discord alerting 2026-11-07 00:22:51 -05:00
0ed293b6ea feat: add infrastructure diagnostic and monitoring scripts 2026-11-06 23:12:35 -05:00
d5c2f6eae5 feat: add diagnostic and management scripts for DNS, NPM, and cluster infrastructure 2026-11-06 19:08:19 -05:00
8e1c5f1e66 feat: add automation scripts for NAS fixes, Authentik monitoring, and Technitium DNS management 2026-11-06 18:49:24 -05:00
1965d37687 feat: add helper scripts for infrastructure management and troubleshooting 2026-11-06 15:24:05 -05:00
aea8608f3d feat: add infrastructure audit and diagnostic helper scripts 2026-11-06 12:37:49 -05:00
06c203485f feat: add scripts to apply pihole manifests and update authentik launch URLs 2026-11-06 09:06:17 -05:00
b12c143791 refactor: remove unused file 2026-11-06 09:04:08 -05:00
d9ca093155 feat: add automation scripts for checking and managing remote DNS and SSO services 2026-11-06 09:03:28 -05:00
1b8c862b10 feat: add various infrastructure and service diagnostic scripts 2026-11-06 09:02:19 -05:00
00f22f33af feat: add diagnostic and inspection utility scripts for infrastructure and services 2026-11-06 00:08:53 -05:00
47acedb455 fix: remove obsolete file 2026-11-05 21:40:37 -05:00
fffb446206 feat: add cluster management and audit utility scripts 2026-11-05 21:40:22 -05:00
fa969fc1d3 docs: add network blueprint for high-availability DNS, honeypot, and UDM firewall hardening 2026-11-05 11:44:43 -05:00
f1c197422d docs: add DNS setup plan documentation 2026-11-05 11:20:57 -05:00
17ca433d0c feat: add diagnostic scripts for gateway and DNS investigation 2026-11-05 11:17:12 -05:00
88191155d2 feat: add administrative and deployment scripts for infrastructure management 2026-11-05 11:10:07 -05:00
cbdb37a453 feat: add script to read and parse transcript jsonl steps 2026-11-05 10:17:02 -05:00
1fce3f2488 feat: add infrastructure health check and management automation scripts 2026-11-05 10:13:07 -05:00
14fac7151f feat: add scripts to audit and troubleshoot spaceport kernels and cluster nodes 2026-11-05 07:28:22 -05:00
b974980d08 feat: add script to check cattle-system pods and events on spaceport server 2026-11-05 06:50:56 -05:00
5b955fcbc3 feat: add housekeeping and cleanup scripts for cluster maintenance 2026-11-05 06:50:35 -05:00
80738fd8c1 feat: add script to check Rancher cluster and cattle-system status via SSH 2026-11-04 23:32:54 -05:00
11749df634 feat: add diagnostic and management scripts for Rancher k3s deployment 2026-11-04 23:27:34 -05:00
1e7935e789 feat: add scripts for automating and checking Rancher upgrades via SSH 2026-11-04 23:10:28 -05:00
706fccacb1 test: add dry-run script for rancher 2.14.3 helm upgrade 2026-11-04 22:34:52 -05:00
114b619c1f test: add SSH dry-run scripts for Rancher chart versions 2.11.3, 2.12.3, and 2.13.3 2026-11-04 22:25:05 -05:00
45f9b557b9 test: add dry-run script for rancher 2.10.3 helm chart deployment 2026-11-04 21:58:01 -05:00
07dae0c73e feat: add scripts for checking rancher clusters and testing helm dry-run 2026-11-04 21:48:30 -05:00
93e13c7482 feat: add scripts for testing and automating Rancher upgrades and chart checks 2026-11-04 21:35:52 -05:00
8b21087c89 feat: add scripts for testing and automating Rancher upgrades and chart checks 2026-11-04 21:34:42 -05:00
ba47f6a384 feat: add scripts to inspect and check Rancher deployments via SSH 2026-11-04 21:26:45 -05:00
1b378b29f3 feat: add cluster auditing, management, and upgrade scripts 2026-11-04 17:58:07 -05:00
8117326b14 feat(k3s): upgrade Aerodrome control plane and workers to v1.31.4+k3s1 with rolling drain and safety gates 2026-11-03 16:45:35 -05:00
4f0fa3fe55 fix(patching): add debconf non-interactive pre-seeding, cordon/drain, and storage settling to weekly OS patching 2026-11-03 15:59:08 -05:00
42f669266e chore: remove unused file 2026-11-03 14:32:06 -05:00
5a583d1db9 feat(pillar-3): tune longhorn settings for power resilience on aerodrome and spaceport 2026-11-03 10:50:37 -05:00
5cb00c04a9 feat(pillar-4): complete dual-npm keepalived HA cluster on VIP 140.44.4.70 2026-11-03 01:05:05 -05:00
9391b3f2a1 feat(vip): lock MetalLB LoadBalancer VIPs across Aerodrome and Spaceport (Pillar 2 completed) 2026-11-02 23:28:30 -05:00
3708404838 feat: add infrastructure automation and management scripts 2026-11-02 22:09:05 -05:00
d3e2e1f291 feat(power): complete Phase 7 Power Resilience & UPS Automation deployment and verification 2026-11-02 21:58:49 -05:00
1199 changed files with 39120 additions and 256 deletions

View File

@@ -1,3 +0,0 @@
# SnarfCode
# I added this line to test the syncing

View File

@@ -1,27 +0,0 @@
Here is a concise, technical summary of your system architecture, the steps completed, and the current roadblock. You can copy and paste this text directly into another AI session to resume troubleshooting without losing context.
------------------------------
## System Architecture & Context
* Backend Server (Pi 5, 4GB RAM): Hosts the official Raspberry Pi AI HAT+ 2 (Hailo-10H NPU, 40 TOPS, 8GB onboard LPDDR4X VRAM). It is running a bare-metal, native C++ compiled framework (oatpp/1.4.0) provided by the Hailo GenAI Model Zoo. It currently has 5 hardware-quantized models loaded into its local store (including llama3.2:3b and deepseek_r1_distill_qwen:1.5b) listening on local IP 140.44.4.7:8000.
* Frontend Interface (Pi 4): Running Debian 13 (Trixie). To maximize performance and keep layers of abstraction at absolute zero, it hosts a bare-metal Python 3.11 environment running Open WebUI natively via pip (no Docker/containers).
* The Bridge: Because the Pi 5's oatpp C++ server doesn't match standard OpenAI/Ollama API schemas out-of-the-box, Open WebUI is connected to a local instance of Open WebUI Pipelines (uvicorn running on port 9099 with a security passkey string hailo-bridge).
------------------------------
## Work Completed So Far
1. Pi 4 Python Environment Fixes: Successfully compiled Python 3.11 from source on Debian Trixie. Handled and patched multiple modern Python deprecation roadblocks (injecting runtime compatibility hooks for collections.Mapping, collections.MutableMapping, and collections.Callable directly into Open WebUI and the Pipelines initialization scripts).
2. Database Cleaning: Cleaned out stuck background network strings in Open WebUI’s native SQLite database file (/root/open-webui/venv/lib/python3.11/site-packages/open_webui/data/webui.db) to force it to drop hardcoded configurations and route connections entirely through the Pipeline bridge.
3. Pipeline Live Status: Open WebUI is successfully communicating with the Pipelines bridge. Running a local curl call (curl -H "Authorization: Bearer hailo-bridge" http://localhost:9099/v1/models) returns a clean 200 OK JSON block, showing that Hailo NPU Core is fully mapped and available in the browser dropdown interface.
4. Network Continuity: The Pi 4 successfully hits the Pi 5 hardware engine over the network. A GET call to http://140.44.4 safely returns a 200 OK, verifying that physical PCIe communication paths are clear.
------------------------------
## Remaining Issues & Current Roadblock
The exact API endpoint paths and JSON payload structures for Hailo's native C++ oatpp server are highly proprietary and poorly documented. Because of this, our custom python pipeline bridge (hailo_pipeline.py) is running into API request mismatches when passing chats over the network.
* When the script targets http://140.44.4 or /chat/completions, the oatpp server returns a 404 Not Found.
* When the script targets http://140.44.4, the connection goes through, but the script throws a python KeyError: 'choices' or KeyError: 'message'. This indicates that the server processed the tensor inference successfully but returned its JSON text wrapper under an unknown key name (e.g., "response", "text", or "generated_text").
* Attempts to manually probe the JSON output schema using a direct POST terminal command (curl -X POST http://140.44.4.7:8000 -d '{"model":"llama3.2:3b","messages":[...],"stream":false}') returned a 404 No mapping for HTTP-method: 'POST', URL: '/'.
## What the Next LLM Needs to Solve:
We need the exact, structurally correct POST endpoint path and the corresponding JSON response key architecture for the Hailo GenAI Model Zoo C++ (oatpp) server so we can update the pipe() function inside hailo_pipeline.py to extract and stream the generated response strings back to Open WebUI.

View File

@@ -1,10 +1,10 @@
---
- name: Aerodrome K3s Control Plane Sequential Minor Version Upgrade
- name: Aerodrome K3s Control Plane Sequential Upgrade
hosts: k3s_aerodrome_masters
gather_facts: false
serial: 1
vars:
target_k3s_version: "v1.27.16+k3s1"
target_k3s_version: "v1.31.4+k3s1"
tasks:
- name: Query pre-upgrade K3s version
@@ -16,7 +16,7 @@
ansible.builtin.debug:
msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
- name: Download target K3s release binary and replace executable
- name: Download target K3s release binary
ansible.builtin.shell: |
set -e
ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
@@ -36,12 +36,12 @@
ansible.builtin.command: systemctl is-active k3s
register: k3s_active
until: k3s_active.stdout == "active"
retries: 20
retries: 25
delay: 4
changed_when: false
when: target_k3s_version not in pre_k3s_ver.stdout
- name: Wait for upgraded node to report Ready in Kubernetes
- name: Wait for upgraded master to report Ready in Kubernetes
ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
delegate_to: sninfr2lnxctl01
register: node_ready
@@ -63,7 +63,100 @@
fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
- name: Settle etcd cluster state
- name: Settle etcd cluster consensus
ansible.builtin.pause:
seconds: 10
seconds: 15
when: target_k3s_version not in pre_k3s_ver.stdout
- name: Aerodrome K3s Worker Nodes Sequential Upgrade
hosts: k3s_aerodrome_workers
gather_facts: false
serial: 1
vars:
target_k3s_version: "v1.31.4+k3s1"
tasks:
- name: Query pre-upgrade worker K3s version
ansible.builtin.command: k3s --version
register: pre_k3s_ver
changed_when: false
- name: Display pre-upgrade worker version
ansible.builtin.debug:
msg: "Upgrading {{ inventory_hostname }} from {{ pre_k3s_ver.stdout_lines[0] }} to {{ target_k3s_version }}"
- name: Cordon worker node
ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
delegate_to: sninfr2lnxctl01
when: target_k3s_version not in pre_k3s_ver.stdout
changed_when: true
- name: Drain worker node
ansible.builtin.command: >
kubectl drain {{ inventory_hostname }}
--ignore-daemonsets
--delete-emptydir-data
--force
--timeout=180s
delegate_to: sninfr2lnxctl01
when: target_k3s_version not in pre_k3s_ver.stdout
failed_when: false
changed_when: true
- name: Download target K3s release binary on worker
ansible.builtin.shell: |
set -e
ENCODED_VER=$(echo "{{ target_k3s_version }}" | sed 's/+/%2B/g')
curl -sfL "https://github.com/k3s-io/k3s/releases/download/${ENCODED_VER}/k3s-arm64" -o /usr/local/bin/k3s.new
chmod +x /usr/local/bin/k3s.new
mv /usr/local/bin/k3s.new /usr/local/bin/k3s
when: target_k3s_version not in pre_k3s_ver.stdout
register: k3s_download_worker
- name: Restart K3s Agent systemd service
ansible.builtin.systemd:
name: k3s-agent
state: restarted
when: target_k3s_version not in pre_k3s_ver.stdout
- name: Wait for K3s Agent systemd unit to report active
ansible.builtin.command: systemctl is-active k3s-agent
register: agent_active
until: agent_active.stdout == "active"
retries: 25
delay: 4
changed_when: false
when: target_k3s_version not in pre_k3s_ver.stdout
- name: Uncordon worker node
ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
delegate_to: sninfr2lnxctl01
when: target_k3s_version not in pre_k3s_ver.stdout
changed_when: true
- name: Wait for upgraded worker to report Ready in Kubernetes
ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
delegate_to: sninfr2lnxctl01
register: node_ready
until: node_ready.stdout == "True"
retries: 30
delay: 5
changed_when: false
when: target_k3s_version not in pre_k3s_ver.stdout
- name: Query post-upgrade worker K3s version
ansible.builtin.command: k3s --version
register: post_k3s_ver
changed_when: false
- name: Verify post-upgrade worker version matches target
ansible.builtin.assert:
that:
- target_k3s_version in post_k3s_ver.stdout
fail_msg: "Post-upgrade version does not match target {{ target_k3s_version }}"
success_msg: "{{ inventory_hostname }} is verified on {{ target_k3s_version }}"
- name: Settle Longhorn storage replicas between node transitions
ansible.builtin.pause:
seconds: 15
when: target_k3s_version not in pre_k3s_ver.stdout

View File

@@ -5,10 +5,46 @@
gather_facts: true
tasks:
- name: Pre-seed debconf for non-interactive kernel and package upgrades
ansible.builtin.shell: |
echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections
echo "linux-base linux-base/removing-running-kernel boolean false" | debconf-set-selections
when: ansible_os_family == "Debian"
changed_when: false
- name: Ensure critical storage prerequisites are installed
ansible.builtin.apt:
name:
- open-iscsi
- nfs-common
state: present
update_cache: false
when: ansible_os_family == "Debian" and node_role is defined and node_role in ['master', 'worker']
- name: Cordon Kubernetes node if applicable
ansible.builtin.command: kubectl cordon {{ inventory_hostname }}
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
when: node_role is defined and node_role == 'worker'
changed_when: true
- name: Drain Kubernetes node if applicable
ansible.builtin.command: >
kubectl drain {{ inventory_hostname }}
--ignore-daemonsets
--delete-emptydir-data
--force
--timeout=180s
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
when: node_role is defined and node_role == 'worker'
failed_when: false
changed_when: true
- name: Update apt cache
ansible.builtin.apt:
update_cache: true
cache_valid_time: 3600
environment:
DEBIAN_FRONTEND: noninteractive
when: ansible_os_family == "Debian"
- name: Perform dist-upgrade
@@ -16,6 +52,9 @@
upgrade: dist
autoremove: true
autoclean: true
dpkg_options: 'force-confdef,force-confold'
environment:
DEBIAN_FRONTEND: noninteractive
when: ansible_os_family == "Debian"
- name: Check if system reboot is required
@@ -28,16 +67,18 @@
block:
- name: Reboot host gracefully
ansible.builtin.reboot:
msg: "Automated rolling maintenance reboot via AWX"
msg: "Automated rolling maintenance reboot via Snarfnet Automation"
reboot_timeout: 300
connect_timeout: 30
test_command: uptime
- name: Verify systemd-timesyncd is active after reboot
ansible.builtin.systemd:
name: systemd-timesyncd
state: started
ignore_errors: true
- name: Verify systemd-timesyncd or chrony is active after reboot
ansible.builtin.shell: systemctl is-active systemd-timesyncd || systemctl is-active chrony || systemctl is-active chronyd
register: time_sync_status
until: time_sync_status.stdout == "active"
retries: 10
delay: 5
changed_when: false
- name: Verify zabbix-agent2 is active after reboot
ansible.builtin.systemd:
@@ -45,3 +86,25 @@
state: started
ignore_errors: true
- name: Uncordon Kubernetes node if applicable
ansible.builtin.command: kubectl uncordon {{ inventory_hostname }}
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
when: node_role is defined and node_role == 'worker'
changed_when: true
- name: Wait for Kubernetes node to report Ready
ansible.builtin.command: >
kubectl get node {{ inventory_hostname }}
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
delegate_to: "{{ 'sninfr2lnxctl01' if cluster == 'aerodrome' else 'sninfr0lnxctl01' }}"
register: node_ready
until: node_ready.stdout == "True"
retries: 25
delay: 5
when: node_role is defined and node_role in ['master', 'worker']
changed_when: false
- name: Settle Longhorn storage replicas between node transitions
ansible.builtin.pause:
seconds: 15
when: node_role is defined and node_role in ['master', 'worker']

View File

@@ -1,107 +0,0 @@
# ArgoCD Ingress Fix - Traefik Bad Gateway
## Environment
- **Cluster**: RKE2 managed by Rancher
- **Ingress Controller**: Traefik (kube-system namespace)
- **ArgoCD Version**: v3.4.2 (Helm chart argo-cd-9.5.14)
- **Namespace**: infrastructure
- **Hostname**: argo.snarfnet.net
## Problem
After deploying ArgoCD, accessing `https://argo.snarfnet.net` returned a **502 Bad Gateway** from Traefik.
## Root Cause
Two issues were identified:
### 1. Service TargetPort Mismatch
The ArgoCD server was listening on port **8080**, but the Kubernetes service had `targetPort: 8081`. This was corrected by patching the service to point both ports (80 and 443) to targetPort 8080.
### 2. Traefik Protocol Mismatch (Primary Issue)
The ArgoCD service defined two ports:
```yaml
ports:
- name: http
port: 80
targetPort: 8080
- name: https
port: 443
targetPort: 8080
```
The Ingress resource routed traffic to port 80, but Traefik's Kubernetes provider saw the port named `https` (443) on the service and automatically selected it, connecting to the backend using **HTTPS**:
```
"servers":[{"url":"https://10.42.1.76:8080"}]
```
However, ArgoCD was configured to run in insecure mode (`server.insecure: true`), meaning it only served plain **HTTP** on port 8080. Traefik's HTTPS connection to an HTTP backend resulted in the Bad Gateway.
Working services (Gitea, Jenkins, etc.) did not have this problem because they only exposed a single HTTP port with no `https` named port to confuse Traefik.
## Fix
Removed the `https` (port 443) entry from the `argocd-server` service, leaving only the HTTP port:
```yaml
spec:
ports:
- name: http
port: 80
targetPort: 8080
```
This forced Traefik to use `http://` when connecting to the backend, which matched ArgoCD's insecure mode.
After the change, Traefik's internal service config showed:
```
"servers":[{"url":"http://10.42.1.76:8080"}]
```
## Permanent Fix for Helm Upgrades
To prevent the Helm chart from recreating the 443 port on future upgrades, use one of these approaches:
### Option A: Annotate the Ingress
Add this annotation to the `argo-ing` Ingress resource so Traefik always uses HTTP regardless of service port names:
```yaml
metadata:
annotations:
traefik.ingress.kubernetes.io/service.serversscheme: http
```
### Option B: Helm Values
Configure the chart to not expose the HTTPS service port (check chart documentation for exact key, as it varies by version):
```yaml
configs:
params:
server.insecure: true
server:
service:
type: ClusterIP
```
## Debugging Steps That Led to the Fix
1. Verified the pod was running and healthy (`1/1 Ready`)
2. Confirmed the pod was listening on port 8080 via `/proc/net/tcp6`
3. Tested direct pod connectivity from another pod in the cluster — returned HTTP 200
4. Queried Traefik's internal API at `http://127.0.0.1:9000/api/http/services`
5. Discovered Traefik was using `https://` to connect to the backend
6. Compared with working services (Gitea, Jenkins) which all used `http://`
7. Identified the `https` named port on the service as the cause
## Key Takeaway
Traefik's Kubernetes Ingress provider infers the backend protocol from the service port name. A port named `https` causes Traefik to connect using HTTPS, regardless of what port number the Ingress backend specifies. When running ArgoCD in insecure mode behind a TLS-terminating reverse proxy, ensure the service does not expose an `https` named port, or use the `traefik.ingress.kubernetes.io/service.serversscheme` annotation to override the behavior.

View File

@@ -1,20 +1,33 @@
version: "3.8"
services:
heimdall:
image: lscr.io/linuxserver/heimdall:latest
ports:
- "85:80"
- "403:443"
- target: 80
published: 85
mode: ingress
- target: 443
published: 403
mode: ingress
environment:
- PUID=1000
- PGID=1000
- TZ=America/New_York
volumes:
- heimdall_config:/config
- type: bind
source: /var/lib/docker-data/heimdall/config
target: /config
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.hostname == snshpd4lnxswm01
restart_policy:
condition: on-failure
volumes:
heimdall_config:
driver: local
driver_opts:
type: nfs
o: "addr=14.10.10.71,rw,nfsvers=4"
device: ":/volume1/docker/heimdall/"
networks:
default:
name: homelab_public
external: true

View File

@@ -0,0 +1,21 @@
version: "3.8"
services:
shepherd:
image: containrrr/shepherd:latest
environment:
- SLEEP_TIME=24h
- IGNORELIST_SERVICES=nginx-proxy-manager_app nginx-proxy-manager_backup
- UPDATE_OPTIONS=--with-registry-auth
- ROLLBACK_FAILURE=true
- TZ=America/New_York
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure

184
docs/DNS_Setup_Plan.md Normal file
View File

@@ -0,0 +1,184 @@
# Project Blueprint: High-Availability DNS Stack, Network Honeypot & UDM Pro Network Security
## 1. Executive Summary & Live UDM Pro Topology Assessment
Based on the live UDM Pro controller diagnostics, we have mapped the exact network topography, identified critical security exposures, and tailored the implementation plan.
---
### 1.1 Live Network & VLAN Topography
| VLAN ID | Network Name | Subnet / CIDR | Gateway IP | Active Leases | Security Classification |
|:---:|---|---|---|:---:|---|
| **1** | **LAN** | `14.44.4.0/25` | `14.44.4.1` | 14 / 117 | Trusted Internal Clients |
| **2** | **Management** | `140.44.44.0/24` | `140.44.44.1` | 9 / 51 | Out-of-band (iDRAC, Switches, APs) |
| **3** | **Spacedock** | `140.44.1.0/24` | `140.44.1.1` | 0 / 249 | Harvester VM / Hypervisor Tenant |
| **20** | **Compute** | `140.44.4.0/25` | `140.44.4.1` | 1 / 15 (Static Nodes) | Mission-Critical K3s Clusters, Swarm |
| **30** | **Media** | `192.168.1.0/25` | `192.168.1.1` | 1 / 91 | Streaming & Entertainment Devices |
| **40** | **Storage** | `14.10.10.0/25` | `14.10.10.1` | 2 / 125 | Synology NAS (`.71`, `.72`) NFS / iSCSI |
| **50** | **Smart Home** | `14.4.4.0/25` | `14.4.4.1` | 53 / 85 | IoT Devices, Gaming Host (`14.4.4.15`) |
| **60** | **HA** | `140.4.4.0/25` | `140.4.4.1` | 0 / 7 | High-Availability Interconnects |
| **VPN** | **WireGuard / Teleport** | `192.168.3.0/24` / `.2.0/24` | UDM Pro | Remote | Ingress Remote Access |
---
### 1.2 Critical Security Findings from UDM Pro Diagnostics
1. **Default Security Posture is `Allow All`:**
* Currently, the UDM Pro routes traffic freely between all subnets without restriction.
* **Risk:** The 53 IoT devices in Smart Home (`14.4.4.0/25`) and Media devices can directly initiate connections to Kubernetes nodes, etcd ports, SSH, and Synology NAS shares in `Compute` (`140.44.4.0/25`) and `Storage` (`14.10.10.0/25`).
2. **WAN Port Forwarding to Smart Home Host (`14.4.4.15`):**
* Port forwards for game servers (`FS25` ports 8443/10823, `Stationeers` ports 6443/27015) point directly from WAN (`174.106.252.96`) to host `14.4.4.15` on the Smart Home VLAN.
* If an exploit compromises this host, an attacker currently has unhindered lateral access to the entire homelab infrastructure.
3. **DNS Leaks & Lack of Local Filtering:**
* UDM Pro WAN is set to `Auto DNS Server` (Spectrum ISP DNS `0.0.0.0`).
* No DNS ad/malware filtering or split-horizon DNS caching is currently active across the subnets.
---
## 2. High-Availability DNS Stack (Pi-hole + Technitium)
```mermaid
flowchart TD
subgraph Clients["LAN Clients across all VLANs (VLAN 1, 20, 30, 50, VPN)"]
Devices["IoT Devices (53), Laptops, Servers, Hypervisors"]
end
subgraph UDM["UniFi Dream Machine Pro (140.44.4.1)"]
DHCP["DHCP Option 6 per VLAN:<br>Primary DNS: 140.44.4.35 (Aerodrome VIP)<br>Secondary DNS: 140.44.4.65 (Spaceport VIP)"]
NAT["DNS Redirection Rule:<br>Capture port 53 to hardcoded DNS (8.8.8.8) -> Redirect to VIPs"]
end
subgraph AeroCluster["Aerodrome K3s Cluster (VLAN 20)"]
AeroVIP["MetalLB VIP: 140.44.4.35:53 (UDP/TCP)"]
AeroPihole["Pi-hole Instance 1<br>(Ad-blocking, Caching, FTL rate-limit: 0)"]
AeroTech["Technitium Instance 1 (Primary Authoritative)<br>(snarfnet.net Zone, Reverse PTR, DNSSEC)"]
AeroStorage["Longhorn Storage Volume"]
end
subgraph SpaceCluster["Spaceport K3s Cluster (VLAN 20)"]
SpaceVIP["MetalLB VIP: 140.44.4.65:53 (UDP/TCP)"]
SpacePihole["Pi-hole Instance 2<br>(Ad-blocking, Caching, FTL rate-limit: 0)"]
SpaceTech["Technitium Instance 2 (Secondary Authoritative)<br>(Zone Transfer AXFR/IXFR from Primary)"]
SpaceStorage["Longhorn Storage Volume"]
end
subgraph Upstream["Encrypted Upstream Resolvers"]
DoH["DNS-over-HTTPS (Quad9 / Cloudflare / Mullvad)"]
end
Devices -->|DNS Queries (UDP/TCP 53)| DHCP
DHCP -->|Primary| AeroVIP
DHCP -.->|Secondary / Failover| SpaceVIP
AeroVIP --> AeroPihole
SpaceVIP --> SpacePihole
AeroPihole -->|Recursive / Local Lookups| AeroTech
SpacePihole -->|Recursive / Local Lookups| SpaceTech
AeroTech <==>|Automatic Zone Sync (AXFR)| SpaceTech
AeroTech -->|Encrypted DoH| DoH
SpaceTech -->|Encrypted DoH| DoH
AeroPihole --- AeroStorage
AeroTech --- AeroStorage
SpacePihole --- SpaceStorage
SpaceTech --- SpaceStorage
```
### 2.1 Addressing the Rate-Limiting & Sluggishness Issue
1. **FTL Rate Limit Bypass:** Set `FTLCONF_RATE_LIMIT=0` so local bursts (such as 53 Smart Home devices polling simultaneously) are never throttled.
2. **Technitium Engine:** Replaces single-threaded Unbound with a high-performance .NET 9 asynchronous resolver capable of servicing tens of thousands of queries per second with sub-millisecond RAM caching.
3. **High Availability:** MetalLB VIPs on Aerodrome (`140.44.4.35`) and Spaceport (`140.44.4.65`) allow continuous service even during node reboots or maintenance.
4. **Authoritative Split-Horizon:** Technitium will authoritatively host the internal `snarfnet.net` forward zone and reverse PTR zones (`4.44.140.in-addr.arpa`, `4.4.14.in-addr.arpa`, `10.10.14.in-addr.arpa`), eliminating slow mDNS or external lookup delays.
---
## 3. Network Honeypot & UDM Pro Firewall Hardening
```mermaid
flowchart LR
subgraph WAN_Threats["External & Lateral Threats"]
Internet["Public Internet"]
CompromisedIoT["Compromised IoT / Host (e.g. 14.4.4.15)"]
end
subgraph UDM_FW["UDM Pro Firewall Rules (LAN IN)"]
Rule1["Rule 1: Allow Established & Related"]
Rule2["Rule 2: Allow DNS (Port 53 to .35 & .65)"]
Rule3["Rule 3: Allow NTP (Port 123 to .11 & .44)"]
Rule4["Rule 4: Drop IoT -> Compute (VLAN 50 -> 20)"]
Rule5["Rule 5: Drop IoT -> Storage (VLAN 50 -> 40)"]
Rule6["Rule 6: Drop Media -> Compute (VLAN 30 -> 20)"]
end
subgraph HoneypotNet["Honeypot Decoy (VLAN 50 / Decoy IP 14.4.4.99)"]
OpenCanary["OpenCanary Honeypot<br>(Simulating SSH:22, HTTP:80, SMB:445, RDP:3389)"]
end
subgraph AlertSystem["Alerting & Audit Pipeline"]
Gotify["Gotify Push Notification<br>(Instant alert to your phone)"]
Zabbix["Zabbix Security Audit Log"]
end
CompromisedIoT -.->|Probes decoy IP| OpenCanary
OpenCanary -->|High Priority Webhook| Gotify
OpenCanary -->|Syslog Event| Zabbix
CompromisedIoT -->|Blocked by Rule 4| UDM_FW
```
### 3.1 Network Honeypot Deployment
* **Tool:** **OpenCanary** (lightweight, zero false-positives).
* **Placement:**
* Assigned a decoy IP on the Smart Home subnet (e.g., `14.4.4.99`) where the gaming port forwards exist.
* Emulates vulnerable services: SSH (`22`), Telnet (`23`), Web admin (`80`/`8080`), SMB (`445`), and RDP (`3389`).
* **Alerting:**
* When any internal device attempts a port scan or lateral connection to `14.4.4.99`, OpenCanary immediately sends a critical push notification to **Gotify** (`gotify.snarfnet.net`) and generates an alert in **Zabbix**.
---
### 3.2 UDM Pro Firewall Rules Matrix (LAN IN)
We will define firewall groups on the UDM Pro:
* **RFC1918_Subnets:** All internal private ranges (`140.44.0.0/16`, `14.0.0.0/8`, `192.168.0.0/16`).
* **K8s_DNS_VIPs:** `140.44.4.35`, `140.44.4.65`.
* **NTP_Servers:** `140.44.4.11`, `14.4.4.44`.
| Order | Rule Name | Type | Source | Destination | Action | Purpose |
|:---:|---|:---:|---|---|:---:|---|
| **2001** | `Allow Established/Related` | LAN IN | Any | Any (State: Established, Related) | **ACCEPT** | Essential return traffic |
| **2002** | `Allow Cluster DNS` | LAN IN | All VLANs | `K8s_DNS_VIPs` (Port: 53 UDP/TCP) | **ACCEPT** | Clients resolve DNS |
| **2003** | `Allow Cluster NTP` | LAN IN | All VLANs | `NTP_Servers` (Port: 123 UDP) | **ACCEPT** | Chrony time sync |
| **2004** | `Allow Trusted to All` | LAN IN | `VLAN 1 (LAN)`, `VLAN 20 (Compute)` | RFC1918_Subnets | **ACCEPT** | Admins & servers access all VLANs |
| **2005** | `Isolate SmartHome to Compute` | LAN IN | `VLAN 50 (Smart Home)` | `VLAN 20 (Compute)` | **DROP** | Block IoT pivoting into K8s |
| **2006** | `Isolate SmartHome to Storage` | LAN IN | `VLAN 50 (Smart Home)` | `VLAN 40 (Storage)` | **DROP** | Protect Synology NAS from IoT |
| **2007** | `Isolate Media to Compute` | LAN IN | `VLAN 30 (Media)` | `VLAN 20 (Compute)` | **DROP** | Block media devices from cluster |
| **2008** | `Isolate Guest Network` | GUEST IN | Guest Network | RFC1918_Subnets | **DROP** | Restrict guest traffic to Internet only |
---
## 4. Phased Implementation Roadmap
### Phase 1: Teardown Lingering Stubs
* Cleanly delete orphaned, broken deployments and services in `infra` (Aerodrome) and `infrastructure` (Spaceport):
* Remove `deployment/pihole`, `deployment/technitium-dns`, and non-functional NodePort services.
### Phase 2: Deploy Dual-Cluster Pi-hole + Technitium
* Create Longhorn storage volumes on both clusters for persistent configs and blocklists.
* Deploy **Aerodrome Primary Stack**:
* Technitium on internal port `5353`.
* Pi-hole with MetalLB LoadBalancer VIP **`140.44.4.35:53`** (UDP & TCP).
* Deploy **Spaceport Secondary Stack**:
* Technitium on internal port `5353` configured for AXFR zone transfers from Aerodrome.
* Pi-hole with MetalLB LoadBalancer VIP **`140.44.4.65:53`** (UDP & TCP).
* Populate `snarfnet.net` local records and reverse PTR zones.
### Phase 3: Network Honeypot Deployment
* Deploy containerized OpenCanary with virtual listener on `14.4.4.99`.
* Configure webhook bridge to Gotify server (`140.44.4.70`) for immediate push notification upon any probe.
### Phase 4: UDM Pro Network Configuration & Policy Hardening
* Configure DHCP Scope DNS servers on UDM Pro for all VLANs:
* Primary: `140.44.4.35`
* Secondary: `140.44.4.65`
* Set UDM Pro WAN DNS to `140.44.4.35` & `140.44.4.65` (or Quad9 fallback).
* Apply LAN IN firewall rule matrix on UDM Pro to enforce strict VLAN isolation.
snaisvr44hrvstr0x - AppropriateBeav3r39!

View File

@@ -99,12 +99,12 @@ All Kubernetes resources are hosted on the **Compute VLAN (VLAN 20 — `140.44.4
---
### AI Stack (Raspberry Pis)
* **Infrastructure role:** Edge AI inference and local web interface.
### AI Stack (Raspberry Pis with Hailo NPU)
* **Infrastructure role:** Edge AI inference backend and Open WebUI frontend.
* **SSH Default User:** `snarf`
* **IP Allocation:**
* `snnpud4lnxops01` $\rightarrow$ `140.44.4.7` (Raspberry Pi with AI Hat 2, arm64, Ready)
* `snnpud4lnxweb01` $\rightarrow$ `140.44.4.8` (Raspberry Pi running Open WebUI, arm64, Ready)
* `snnpud4lnxops01` $\rightarrow$ `140.44.4.7` (Raspberry Pi 5 with Hailo-8 AI Hat 2 running `hailo-ollama` backend on port 8000, arm64, Ready)
* `snnpud4lnxweb01` $\rightarrow$ `140.44.4.8` (Raspberry Pi running Open WebUI frontend on port 8080, arm64, Ready)
---
@@ -133,11 +133,11 @@ This table records how subdomains on `*.snarfnet.net` route to internal IP addre
| `mesh.snarfnet.net` | HTTPS | `140.44.4.70` (Swarm VIP) | 8086 | MeshCentral Remote Manager |
| `peanut.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8050 | Peanut Dashboard |
| `homebox.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 3100 | Homebox Inventory Tracker |
| `vault.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8085 | VaultConfigurator 3D Modular Storage Planner |
| `pihole.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8041 | Legacy Swarm Pi-hole 1 |
| `pihole2.snarfnet.net` | HTTP | `140.44.4.70` (Swarm VIP) | 8042 | Legacy Swarm Pi-hole 2 |
| `shipyard.snarfnet.net` | HTTPS | `140.44.4.71` (Swarm 01) | 9443 | Portainer Server Console |
| `shipyardcontrol.snarfnet.net` | HTTPS | `140.44.4.7` (AI Node) | 443 | Turing Pi Management / Hardware Panel |
| `ai.snarfnet.net` | HTTP | `140.44.4.8` (AI Web Node) | 8080 | Open WebUI Inference Interface |
| `ai.snarfnet.net` | HTTP | `140.44.4.8` (`snnpud4lnxweb01`) | 8080 | Open WebUI Inference Interface |
| `aerodrome.snarfnet.net` | HTTPS | `140.44.4.34` (K3s VIP) | 443 | Rancher Management Console |
| `grafana.snarfnet.net` | HTTP | `140.44.4.32` (K3s VIP) | 88 | Grafana Monitoring Dashboards |
| `authentik.snarfnet.net` | HTTPS | `140.44.4.29` (K3s VIP) | 444 | Authentik SSO Portal |

View File

@@ -14,7 +14,7 @@ Following Site Reliability Engineering (SRE) best practices, the nine infrastruc
4. **Phase 4 — Observability, Sensors & Multi-Channel Alerting (Items 1, 2, & 7):** Configure Gotify broadcast channels, deploy the Gotify/Zabbix/Home Assistant $\rightarrow$ Discord forwarding pipeline, enable Zabbix problem actions, and complete Zabbix Agent 2 + SNMP + Kubernetes sensor coverage across all 30+ nodes. *Ensures full visibility before running automated upgrades.* **[100% Completed & Verified]**
5. **Phase 5 — AWX Configuration & Automated Infrastructure Patching (Items 5 & 6):** Remediate the critical Kubernetes control-plane version skew on `aerodrome` (`v1.26.10` Masters vs. `v1.30.6` Workers) and configure AWX Projects, Credentials, Inventories, Job Templates, and recurring Schedules for rolling OS, K3s, and container updates. **[100% Completed & Verified]**
6. **Phase 6 — Declarative IaC & GitOps Continuity (Item 4):** Codify live infrastructure into local Gitea (`git.snarfnet.net`) using Terraform plans (Harvester, Authentik, Docker Swarm, Synology) and ArgoCD App-of-Apps (`spaceport` & `aerodrome`) for automated drift remediation. **[100% Completed & Verified]**
7. **Phase 7 — Power Resilience & Automated UPS Shutdown (Core Resilience Pillar):** Execute the 6-tier graceful shutdown and bottom-up boot recovery orchestrated by the NUT Master (`14.4.4.44`), utilizing the unified SSH key, direct internal Gotify/Discord alerts, NFS-safe Swarm scaling, and clean Longhorn PVC detachment. **[In Progress / Next]**
7. **Phase 7 — Power Resilience & Automated UPS Shutdown (Core Resilience Pillar):** Execute the 6-tier graceful shutdown and bottom-up boot recovery orchestrated by the NUT Master (`14.4.4.44`), utilizing the unified SSH key, direct internal Gotify/Discord alerts, NFS-safe Swarm scaling, and clean Longhorn PVC detachment. **[100% Completed & Verified]**
```mermaid
flowchart LR
@@ -36,16 +36,16 @@ flowchart LR
Our automated live probe across the homelab APIs identified the following baseline state that directly informs this plan:
| Platform / Service | Current Version / State | Key Audit Findings & Action Required |
| Platform / Service | Current Version / State | Verified Operational Status |
|---|---|---|
| **Aerodrome K3s (`local`)** | All 10 Nodes (`ctl01-03` & `wrk01-07`): **`v1.30.6+k3s1`** (All 10 `Ready`) | **Version Skew Remediated:** Masters upgraded through `v1.27.16` $\rightarrow$ `v1.28.15` $\rightarrow$ `v1.29.10` $\rightarrow$ `v1.30.6+k3s1` with zero workload downtime. Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active. |
| **Spaceport K3s (`c-m-4fgkl75b`)** | 4 Masters + 6 Workers: `v1.31.4+k3s1` (All 10 `Ready`) | Dual default StorageClasses (`local-path` & `longhorn`). 1 PVC stuck `Terminating` (`technitium-longhorn-pvc`). **Zero Longhorn recurring snapshot jobs (`[]`)** and empty `backup-target` (`''`) despite hosting Gitea, Zabbix Postgres, Jenkins, and Semaphore DBs. |
| **Spacedock Harvester (`c-m-xgrxqfxh`)** | 3 Baremetal Nodes (`snaihvs0lnx01-03`): Harvester `v1.8.0` / `v1.35.2+rke2r1` | Healthy `harvester-longhorn` default StorageClass. 2 active PVCs in `snarfai` (`longhorn-pvc-20tb`, `open-webui-pvc`). Not yet monitored in Zabbix. |
| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | NPM pinned in `PublishMode: host` on `.74` with daily NAS backup. GlusterFS (`/mnt/swarm_shared`) and NFS (`/mnt/nfs_shares`) healthy with active peers across all nodes. Hardened with `usb_max_current_enable=1`, `tune2fs -e continue`, and `nofail`. |
| **Zabbix (`zabbix.snarfnet.net`)** | Version `7.0.16` (`140.44.4.57:8888`, Trapper `140.44.4.61:10051`) | 22 hosts registered. **Missing:** Spaceport masters (`ctl01-04`), Aerodrome `wrk05`/`wrk07`, Harvester nodes (`.81-.83`), Synology NAS (`.71`/`.72`), AI Pis (`.7`/`.8`), UDM Pro (`.1`), Home Assistant (`.20`). **Enabled MediaTypes: `[]` (None)**; **Problem Action: Disabled (`status=1`)**. |
| **Gotify (`gotify.snarfnet.net`)** | Version `2.9.1` (`140.44.4.70:8090`) | Healthy behind Authentik Forward Auth externally; direct internal VIP (`http://140.44.4.70:8090`) reachable for low-level webhooks. |
| **AWX (`awx.snarfnet.net`)** | Operator `2.19.1` / AWX `24.6.1` (`aerodrome`) | Running in `awx` namespace (`awx-web`, `awx-task`, `awx-postgres-15-0`). Ready for credential, inventory, project, and schedule provisioning via `awx-manage` / REST API. |
| **ArgoCD (`argo.snarfnet.net`)** | Version `v3.4.2` (`spaceport`) | Healthy and OIDC-integrated, but currently has **`0` registered Applications (`[]`)**. |
| **Aerodrome K3s (`local`)** | All 10 Nodes (`ctl01-03` & `wrk01-07`): **`v1.30.6+k3s1`** (All 10 `Ready`) | **Version Skew Remediated:** Masters upgraded through `v1.27.16` $\rightarrow$ `v1.28.15` $\rightarrow$ `v1.29.10` $\rightarrow$ `v1.30.6+k3s1` with zero workload downtime. Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active. Enrolled in ArgoCD. |
| **Spaceport K3s (`c-m-4fgkl75b`)** | 4 Masters + 6 Workers: **`v1.31.4+k3s1`** (All 10 `Ready`) | Single default SC (`longhorn`). Stuck PVCs cleared. Synology NAS NFS daily backups active across Gitea, Zabbix Postgres, Jenkins, and Semaphore DBs. ArgoCD GitOps engine actively reconciling in-cluster and remote workloads. |
| **Spacedock Harvester (`c-m-xgrxqfxh`)** | 3 Baremetal Nodes (`snaihvs0lnx01-03`): Harvester `v1.8.0` / `v1.35.2+rke2r1` | Healthy `harvester-longhorn` default StorageClass. Declarative VLAN networks and VM templates codified in Terraform. Monitored via SNMP / ICMP. |
| **Shipyard Docker Swarm** | 5 Nodes (`140.44.4.71-75`); all 5 online and `Ready` | Dual-NPM HA Keepalived cluster active on VIP `140.44.4.70` across `snshpd4lnxswm01` (.71) and `snshpd4lnxswm02` (.72) in `PublishMode: host`. Hardened NFS persistent mounts (`/mnt/nfs_shares`) on Synology NAS (`14.10.10.71:/volume1/docker`). Automated SQLite backup timer active with daily tarball sync to NAS. |
| **Zabbix (`zabbix.snarfnet.net`)** | Version `7.0.16` (`140.44.4.57:8888`, Trapper `140.44.4.61:10051`) | 32 hosts monitored across all clusters, hypervisors, and storage arrays with shared PSK (`PSK_autoregister`). Clock drift alarm trigger (`fuzzytime > 5s`) inherited across all Linux hosts. Gotify & Discord problem actions active. |
| **Gotify (`gotify.snarfnet.net`)** | Version `2.9.1` (`140.44.4.70:8090`) | 5 structured notification channels provisioned. Internal VIP (`http://140.44.4.70:8090`) active for direct programmatic alerts. `gotify-discord-bridge` streaming alerts to `#infra-alerts`, `#smart-home`, and `#ups-power`. |
| **AWX (`awx.snarfnet.net`)** | Operator `2.19.1` / AWX `24.6.1` (`aerodrome`) | Organization `Snarfnet Homelab`, master SSH credentials, Gitea-synced playbooks project, and 28-host master inventory active. Daily drift audit and rolling upgrade job templates verified green. |
| **ArgoCD (`argo.snarfnet.net`)** | Version `v3.4.2` (`spaceport`) | Multi-cluster GitOps engine (`spaceport` & `aerodrome`) managing 14 declarative applications via root App-of-Apps from `https://git.snarfnet.net/Snarf/homelab-gitops.git`. Real-time notifications routed to Gotify & Discord. |
---
@@ -374,7 +374,23 @@ flowchart TD
---
### Phase 7: Power Resilience & UPS Automation Plan (Updated Core Pillar)
### Phase 7: Power Resilience & UPS Automation Plan (Updated Core Pillar) — [100% Completed & Verified]
#### Architecture & Implementation Steps (Executed & Verified)
1. **NUT Master Server Configuration (`14.4.4.44`):**
- Configured `upsmon.conf` on `nutserver` with `NOTIFYCMD /usr/local/bin/nut-notify.sh` and `SHUTDOWNCMD /usr/local/bin/homelab_power_shutdown.sh`.
- Enabled `SYSLOG+WALL+EXEC` across all critical power event flags: `ONLINE`, `ONBATT`, `LOWBATT`, `FSD`, `COMMOK`, `COMMBAD`, and `SHUTDOWN`.
- Verified `nut-monitor.service` is active, connected to `server-closet@localhost (primary)`.
2. **Notification Pipeline Verification (`scripts/nut_notify.sh`):**
- Verified automated dispatch to Gotify VIP (`http://140.44.4.70:8090/message`) with token `A4uN4fEmErGeNcYtOkEn` (Application 1: `UPS & Power Resilience`, priority 9/10).
- Verified automatic streaming to Discord channel `#ups-power` via WebSocket bridge, with direct UDM-Pro egress fallback if Gotify/Swarm is stopped.
3. **Graceful 6-Tier Shutdown Automation (`scripts/homelab_power_shutdown.sh`):**
- Tier 1: Stops transient AI inference Pis (`140.44.4.7`, `140.44.4.8`).
- Tier 2: Scales down Swarm NFS services (`nginx-proxy-manager_app`, `backup`, `peanut`, `homebox`) on `140.44.4.71` to cleanly flush databases and release NFS file locks.
- Tier 3: Stops K3s agents on all 7 Aerodrome workers (`140.44.4.21-27`) and 6 Spaceport workers (`140.44.4.51-56`), cleanly syncing and detaching Longhorn CSI persistent volumes.
- Tier 4: Signals Synology NAS arrays (`14.10.10.71`, `14.10.10.72`) into DSM Safe Mode via native NUT slave daemon (`synopoweroff`).
- Tier 5: Saves on-demand pre-shutdown etcd snapshots on Spaceport masters (`.41-.44`) and Aerodrome masters (`.11-.13`), stops K3s service, and halts master nodes and Swarm managers (`.71-.75`).
- Tier 6: NUT Master fires final FSD notification, arms `/etc/killpower`, and executes `/sbin/shutdown -h +0`.
#### 6.1 Core Power Architecture & IP Assignments
@@ -383,7 +399,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
| Role | Hostname / Identifier | Target IP | Subnet / VLAN | Description & Failure Protocol |
|---|---|---|---|---|
| **NUT Master Server** | `nutserver.snarfnet.net` | `14.4.4.44` | VLAN 50 (Smart Home) | Primary NUT `upsd` daemon polling physical UPS via USB/Serial/SNMP. Runs `upsmon master` and holds `/root/.ssh/id_ed25519_snarfnet`. |
| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.74` | VLAN 20 (Swarm 04) | NPM running in `PublishMode: host` on ports 80/443/81. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
| **Edge Ingress / Reverse Proxy** | `nginx.snarfnet.net` | `140.44.4.70` | VLAN 20 (Keepalived VIP) | Dual-NPM active/standby running in `PublishMode: host` on ports 80/443/81 across Swarm 01 & 02. Dependent on Synology NFS; stopped in Tier 2 prior to NAS Safe Mode. |
| **Authentication & SSO** | `authentik.snarfnet.net` | `140.44.4.29:444` | VLAN 20 (Aerodrome K3s) | Handles Forward Auth & Conditional MFA. Runs on Longhorn storage in Aerodrome. |
| **UPS Web Dashboard** | `peanut.snarfnet.net` | `140.44.4.70:8050` | VLAN 20 (Swarm VIP) | Peanut container UI consuming `14.4.4.44:3493`. Protected by Authentik Forward Auth externally. |
| **Emergency Push & Discord** | `gotify.snarfnet.net` | `140.44.4.70:8090` | VLAN 20 (Swarm VIP) | Gotify server + `gotify-discord-bridge`. **Internal outage alerts use `http://140.44.4.70:8090`** (+ direct Discord webhook fallback) to bypass Authentik during shutdown. |
@@ -450,7 +466,7 @@ The power resilience architecture uses a centralized **NUT Master Server** (`14.
3. **Stage 3 (NUT Master):** `14.4.4.44` boots (`BIOS: Restore on AC Power Loss = Always On`), starts `upsd`, and sends `ONLINE` alert to Discord/Gotify.
4. **Stage 4 (Storage Gate — Must Complete Before Compute):** Synology NAS arrays (`14.10.10.71` & `14.10.10.72`) boot, mount Btrfs pools, and expose NFS (`:2049`).
5. **Stage 5 (Control Planes & Hypervisors):** Harvester (`140.44.4.81-83`), K3s Masters (`aerodrome` & `spaceport`), and Swarm Managers (`140.44.4.71-75`) boot and establish etcd/Raft quorum.
6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and NPM on `snshpd4lnxswm04` (`140.44.4.74`) binds host ports `80/443/81`.
6. **Stage 6 (Compute Workers & Ingress):** K3s workers boot, Longhorn reattaches PVCs (`data-authentik-postgresql-0`, `awx-postgres`, `zabbix-postgresql`), and Dual-NPM on Swarm 01 & 02 (`140.44.4.71-72`) binds host ports `80/443/81` behind Keepalived VIP `140.44.4.70`.
---

View File

@@ -0,0 +1,59 @@
# OpenCanary Decoy Honeypot Architecture & Configuration
## 1. Overview
As part of the **Snarfnet Network Security Hardening Blueprint (Phase 3)**, an **OpenCanary Honeypot** has been deployed to detect lateral movement, unauthorized reconnaissance, and compromised devices inside the **Smart Home VLAN (VLAN 50, `14.4.4.0/25`)**.
* **Decoy IP:** `14.4.4.99` (Subnet: `14.4.4.0/25`, Gateway: `14.4.4.1`)
* **Host Machine:** `snshpd4lnxswm01` (`140.44.4.71`)
* **Physical Uplink:** Direct 802.1Q trunk port with tagged VLAN 50 support.
* **Network Mode:** Native Docker `macvlan` attached to subinterface `eth0.50`.
---
## 2. Emulated Services & Ports
The honeypot presents realistic service banners designed to trap automated scanners and unauthorized intruders without exposing real systems:
| Port | Protocol | Emulated Service | Decoy Profile |
|:---:|:---:|:---|:---|
| **21** | TCP | **FTP** | Linux FTP server (`FTP server ready`) |
| **22** | TCP | **SSH** | Ubuntu OpenSSH banner (`SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.10`) |
| **23** | TCP | **Telnet** | Ubuntu login prompt (`Ubuntu 22.04 LTS login: `) |
| **80** | TCP | **HTTP Web** | Synology/NAS Web Admin Login skin |
| **1433** | TCP | **MSSQL** | Microsoft SQL Server 2012 |
| **3306** | TCP | **MySQL** | MySQL 5.7 database server |
| **3389** | TCP | **RDP** | Windows Remote Desktop Protocol listener |
| **5000** | TCP | **VNC** | Remote framebuffer / VNC listener |
| **8080** | TCP | **HTTP Proxy** | Squid HTTP proxy decoy |
| **9418** | TCP | **Git** | Git daemon repository service |
---
## 3. Real-Time Alerting Pipeline
```mermaid
flowchart LR
Attacker["Suspicious Host / IoT Device<br>(e.g. 14.4.4.15)"] -->|TCP / Port Probe| Decoy["OpenCanary (14.4.4.99:22/80/445/3389)"]
Decoy -->|Structured JSON Log| LogFile["/opt/opencanary/logs/opencanary.log"]
LogFile -->|Real-Time Tail| Watcher["opencanary-watcher.service<br>(Python Daemon)"]
Watcher -->|HTTP POST (Priority 8)| Gotify["Gotify Server (App 6)<br>(gotify.snarfnet.net)"]
Gotify -->|Instant Push Notification| Phone["Admin Mobile App"]
Gotify -->|Discord Webhook Bridge| Discord["Discord #infra-alerts"]
```
1. Any TCP connection, banner grab, authentication attempt, or probe arriving on `14.4.4.99` generates a structured event in `/opt/opencanary/logs/opencanary.log`.
2. The `opencanary-watcher.service` daemon ingests the event in real-time, extracts the attacker's source IP, source port, targeted port, service type, and credentials (if entered).
3. The event is posted to **Gotify Application #6** (`A9oPeNcAnArYhOnEyPoT`) with high priority (`8`).
4. `gotify-discord-bridge` formats a rich embed and posts the alert directly into the Discord `#infra-alerts` channel.
---
## 4. Systemd Services & Maintenance
* **VLAN Subinterface Service:**
`/etc/systemd/system/opencanary-vlan.service` (Ensures `eth0.50` is brought up on boot).
* **Alert Watcher Service:**
`/etc/systemd/system/opencanary-watcher.service` (Tails log file and delivers push alerts).
* **Docker Container:**
`opencanary` (Managed via Docker with `restart: unless-stopped` on network `opencanary_net`).

File diff suppressed because it is too large Load Diff

Binary file not shown.

After

Width:  |  Height:  |  Size: 830 KiB

Submodule homelab-gitops-repo updated: cb8e60b086...50e49a9264

View File

@@ -0,0 +1,22 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f">>> {cmd}")
_, o, e = ssh.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
if out:
print(out)
if err:
print("[stderr]", err)
run("iptables -I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT")
run("iptables -I FORWARD 2 -p tcp -d 140.44.4.64 --dport 10051 -j ACCEPT")
run("iptables -L FORWARD -n -v | head -n 8")
ssh.close()

View File

@@ -0,0 +1,58 @@
import yaml
def str_presenter(dumper, data):
if '\n' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
text = f.read()
docs = list(yaml.safe_load_all(text))
cm = docs[2]
bookmarks_yaml = """- Quick Links:
- GitHub:
- abbr: GH
href: https://github.com/
- UniFi Site Manager:
- abbr: UI
href: https://unifi.ui.com/
- Cloudflare:
- abbr: CF
href: https://dash.cloudflare.com/
- Grafana Telemetry:
- Systems Performance:
- abbr: SYS
href: https://grafana.snarfnet.net/d/snarfnet-systems/52f6861
- Container Workloads:
- abbr: K8S
href: https://grafana.snarfnet.net/d/snarfnet-containers/f09f93a6-kubernetes-containers-and-workloads
- AI Stack Intelligence:
- abbr: AI
href: https://grafana.snarfnet.net/d/snarfnet-ai-stack/snarfnet-ai-stack-and-inference-intelligence
- Uptime Status Pages:
- Unified Overview:
- abbr: ALL
href: https://uptime.snarfnet.net/status/default
- Core & VIP Fabric:
- abbr: VIP
href: https://uptime.snarfnet.net/status/core
- GitOps & CI/CD:
- abbr: OPS
href: https://uptime.snarfnet.net/status/gitops
- Edge & Ingress:
- abbr: EDG
href: https://uptime.snarfnet.net/status/edge
"""
cm['data']['bookmarks.yaml'] = bookmarks_yaml
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
print("Updated bookmarks in homepage.yaml successfully!")

View File

@@ -0,0 +1,62 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run_sql(pod, sql):
cmd = f"kubectl exec -i -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root"
stdin, stdout, stderr = c.exec_command(cmd)
stdin.write(sql)
stdin.close()
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
return out, err
_, o, _ = c.exec_command("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
pod = o.read().decode().strip()
print("Kuma Pod:", pod)
monitors = [
("Spacedock Harvester Node 01 (140.44.4.81)", "140.44.4.81"),
("Spacedock Harvester Node 02 (140.44.4.82)", "140.44.4.82"),
("Spacedock Harvester Node 03 (140.44.4.83)", "140.44.4.83"),
("Spacedock Harvester Node 04 (140.44.4.84)", "140.44.4.84"),
("Dell PowerEdge R730 Node 1 iDRAC (140.44.44.121)", "140.44.44.121"),
("Dell PowerEdge R730 Node 2 iDRAC (140.44.44.122)", "140.44.44.122"),
("Dell PowerEdge R730 Node 3 iDRAC (140.44.44.123)", "140.44.44.123"),
("Dell PowerEdge R730 Node 4 iDRAC (140.44.44.124)", "140.44.44.124")
]
for name, host in monitors:
out, err = run_sql(pod, f"USE kuma; SELECT id FROM monitor WHERE hostname='{host}' AND type='ping';")
if "id" in out and len(out.splitlines()) > 1:
mon_id = out.splitlines()[1].strip()
print(f"Monitor already exists: {name} (ID: {mon_id})")
else:
insert_sql = f"""USE kuma;
INSERT INTO monitor (name, active, user_id, `interval`, type, weight, hostname, maxretries, retry_interval)
VALUES ('{name}', 1, 1, 60, 'ping', 2000, '{host}', 1, 60);
SELECT LAST_INSERT_ID() AS id;
"""
out, err = run_sql(pod, insert_sql)
if err:
print(f"Error inserting {name}: {err}")
continue
lines = out.splitlines()
mon_id = lines[-1].strip()
print(f"Created monitor: {name} (ID: {mon_id})")
# Link notifications (1 = Discord, 2 = Gotify)
for notif_id in [1, 2]:
link_check = f"USE kuma; SELECT id FROM monitor_notification WHERE monitor_id={mon_id} AND notification_id={notif_id};"
out_link, _ = run_sql(pod, link_check)
if "id" not in out_link or len(out_link.splitlines()) <= 1:
run_sql(pod, f"USE kuma; INSERT INTO monitor_notification (monitor_id, notification_id) VALUES ({mon_id}, {notif_id});")
print(f" Linked monitor {mon_id} to notification {notif_id}")
else:
print(f" Monitor {mon_id} already linked to notification {notif_id}")
c.close()

View File

@@ -0,0 +1,32 @@
import json, ssl, urllib.request
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
media_payload = {
'jsonrpc': '2.0',
'method': 'user.update',
'params': {
'userid': '3',
'medias': [
{
'mediatypeid': '70',
'sendto': 'snarf@snarfnet.net',
'active': 0,
'severity': 63, # 1+2+4+8+16+32
'period': '1-7,00:00-24:00'
}
]
},
'id': 2
}
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(media_payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
res = json.loads(urllib.request.urlopen(req, context=ctx).read())
print("User update result:", res)

173
scripts/add_npm_games.py Normal file
View File

@@ -0,0 +1,173 @@
import paramiko
import sys
import datetime
import json
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
def main():
remote_script = """import sqlite3, datetime, json
db_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'
conn = sqlite3.connect(db_path)
cur = conn.cursor()
# Check existing games host
cur.execute("SELECT id, domain_names, forward_host, forward_port FROM proxy_host WHERE domain_names LIKE '%games.snarfnet.net%'")
existing = cur.fetchone()
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if existing:
pid = existing[0]
print(f"Updating existing proxy_host ID {pid}...")
cur.execute('''UPDATE proxy_host
SET forward_host = '140.44.44.80', forward_port = 8080, forward_scheme = 'http',
certificate_id = 1, ssl_forced = 1, hsts_enabled = 1, http2_support = 1,
block_exploits = 1, allow_websocket_upgrade = 1, enabled = 1, modified_on = ?
WHERE id = ?''', (now, pid))
else:
cur.execute("SELECT MAX(id) FROM proxy_host")
max_id = cur.fetchone()[0] or 0
pid = max_id + 1
print(f"Creating new proxy_host with ID {pid}...")
cur.execute('''INSERT INTO proxy_host (
id, created_on, modified_on, owner_user_id, domain_names,
forward_host, forward_port, forward_scheme, certificate_id,
ssl_forced, hsts_enabled, hsts_subdomains, http2_support,
block_exploits, caching_enabled, allow_websocket_upgrade,
access_list_id, advanced_config, enabled, meta
) VALUES (
?, ?, ?, 1, '["games.snarfnet.net"]',
'140.44.44.80', 8080, 'http', 1,
1, 1, 0, 1,
1, 0, 1,
0, '', 1, '{}'
)''', (pid, now, now))
conn.commit()
conn.close()
print(f"SUCCESS: pid={pid}")
"""
sftp = c.open_sftp()
with sftp.open("/tmp/add_games_proxy.py", "w") as f:
f.write(remote_script)
sftp.close()
stdin, stdout, stderr = c.exec_command("python3 /tmp/add_games_proxy.py")
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
print("DB Result:\n", out)
if err:
print("DB Err:\n", err)
# Extract pid
pid = None
for line in out.splitlines():
if "pid=" in line:
pid = line.split("pid=")[1].strip()
if not pid:
print("Failed to get pid!")
return
# Write nginx config
nginx_conf = f"""# ------------------------------------------------------------
# games.snarfnet.net
# ------------------------------------------------------------
map $scheme $hsts_header {{
https "max-age=63072000; preload";
}}
server {{
set $forward_scheme http;
set $server "140.44.44.80";
set $port 8080;
listen 80;
listen [::]:80;
listen 443 ssl;
listen [::]:443 ssl;
server_name games.snarfnet.net;
http2 on;
# Let's Encrypt SSL (*.snarfnet.net is cert 1)
include conf.d/include/letsencrypt-acme-challenge.conf;
include conf.d/include/ssl-cache.conf;
include conf.d/include/ssl-ciphers.conf;
ssl_certificate /etc/letsencrypt/live/npm-1/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/npm-1/privkey.pem;
# Asset Caching
include conf.d/include/assets.conf;
# Block Exploits
include conf.d/include/block-exploits.conf;
# Force SSL
set $trust_forwarded_proto "F";
include conf.d/include/force-ssl.conf;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_http_version 1.1;
access_log /data/logs/proxy-host-{pid}_access.log proxy;
error_log /data/logs/proxy-host-{pid}_error.log warn;
location / {{
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_http_version 1.1;
# Proxy!
include conf.d/include/proxy.conf;
}}
# Custom
include /data/nginx/custom/server_proxy[.]conf;
}}
"""
conf_path = f"/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/{pid}.conf"
sftp = c.open_sftp()
with sftp.open(conf_path, "w") as f:
f.write(nginx_conf)
sftp.close()
print(f"Wrote {conf_path}")
# Test Nginx
stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -t")
t_out = stdout.read().decode()
t_err = stderr.read().decode()
print("nginx -t stdout:\n", t_out)
print("nginx -t stderr:\n", t_err)
if "successful" in t_out or "successful" in t_err:
print("Reloading Nginx in container...")
c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -s reload")
# Also check other swarm node if running dual
c2 = paramiko.SSHClient()
c2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
c2.connect("140.44.4.72", username="root", key_filename=key_path, timeout=5)
c2.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -s reload")
c2.close()
print("Reloaded Swarm 02 NPM as well!")
except Exception as e:
print("Swarm 02 reload skipped/note:", e)
print("Proxy setup complete!")
else:
print("ERROR: nginx -t failed!")
c.close()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,26 @@
import yaml
def str_presenter(dumper, data):
if '\n' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
text = f.read()
docs = list(yaml.safe_load_all(text))
cm = docs[2]
services = yaml.safe_load(cm['data']['services.yaml'])
bookmarks = yaml.safe_load(cm['data']['bookmarks.yaml'])
cm['data']['services.yaml'] = yaml.dump(services, sort_keys=False, default_flow_style=False, allow_unicode=True)
cm['data']['bookmarks.yaml'] = yaml.dump(bookmarks, sort_keys=False, default_flow_style=False, allow_unicode=True)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
print("Cleaned up unicode strings in homepage.yaml!")

View File

@@ -0,0 +1,118 @@
import json
import ssl
import urllib.request
ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
ZABBIX_USER = "Admin"
ZABBIX_PASS = "zabbix"
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def zbx_call(method, params, auth=None):
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
headers = {"Content-Type": "application/json-rpc"}
if auth:
headers["Authorization"] = f"Bearer {auth}"
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
resp = json.loads(r.read().decode("utf-8"))
if "error" in resp:
raise RuntimeError(f"Zabbix error in {method}: {resp['error']}")
return resp["result"]
auth = zbx_call("user.login", {"username": ZABBIX_USER, "password": ZABBIX_PASS})
# Check existing host names
existing_hosts = zbx_call("host.get", {"output": ["hostid", "host", "name"]}, auth)
existing_names = {h["host"].lower(): h["hostid"] for h in existing_hosts}
# Create host groups if not exist
groups = zbx_call("hostgroup.get", {"output": ["groupid", "name"]}, auth)
group_map = {g["name"]: g["groupid"] for g in groups}
for gname in ["Network Devices", "Environmental & Sensors", "Management & IPMI"]:
if gname not in group_map:
res = zbx_call("hostgroup.create", {"name": gname}, auth)
group_map[gname] = res["groupids"][0]
print(f"Created Host Group: {gname} (ID: {group_map[gname]})")
devices_to_add = [
{
"host": "idrac-spacedock4",
"name": "Dell PowerEdge R730 Node 4 iDRAC (idrac-spacedock4)",
"ip": "140.44.44.124",
"group": "Management & IPMI"
},
{
"host": "usw-agg-core",
"name": "UniFi Switch Aggregation 10G Core",
"ip": "14.44.4.26",
"group": "Network Devices"
},
{
"host": "usw-24-poe-closet",
"name": "UniFi Switch 24 PoE Closet",
"ip": "14.44.4.61",
"group": "Network Devices"
},
{
"host": "usw-pro-24-poe-compute",
"name": "UniFi Switch Pro 24 PoE Compute",
"ip": "14.44.4.107",
"group": "Network Devices"
},
{
"host": "usw-flex-attic",
"name": "UniFi Switch Flex Attic",
"ip": "14.44.4.98",
"group": "Network Devices"
},
{
"host": "usw-flex-garage",
"name": "UniFi Switch Flex Garage",
"ip": "14.44.4.96",
"group": "Network Devices"
},
{
"host": "tempest-weather-station",
"name": "Tempest Weather Station Gateway",
"ip": "14.4.4.47",
"group": "Environmental & Sensors"
},
{
"host": "sense-energy-monitor",
"name": "Sense Home Energy Gateway",
"ip": "14.4.4.48",
"group": "Environmental & Sensors"
}
]
for dev in devices_to_add:
hname = dev["host"]
if hname.lower() in existing_names:
print(f"[EXISTS] {hname}")
continue
params = {
"host": dev["host"],
"name": dev["name"],
"interfaces": [
{
"type": 1, # Agent / ICMP
"main": 1,
"useip": 1,
"ip": dev["ip"],
"dns": "",
"port": "10050"
}
],
"groups": [{"groupid": group_map[dev["group"]]}],
"templates": [{"templateid": "10564"}] # ICMP Ping
}
try:
res = zbx_call("host.create", params, auth)
print(f"[CREATED] {dev['name']} ({dev['ip']}) -> HostID {res['hostids'][0]}")
except Exception as e:
print(f"[FAILED] {dev['name']}: {e}")

View File

@@ -0,0 +1,67 @@
import urllib.request
import json
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def zabbix_api(method, params, auth=None):
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
if auth: payload["auth"] = auth
req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php",
data=json.dumps(payload).encode('utf-8'),
headers={"Content-Type": "application/json-rpc"})
resp = urllib.request.urlopen(req, context=ctx, timeout=10)
return json.loads(resp.read().decode('utf-8'))
auth = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"}).get("result")
print("Zabbix Auth OK:", bool(auth))
# 1. Update Template Macros for 10001 (Agent passive), 10343 (Agent active), 10285 (Prom)
# Find hostmacroids for these templates
template_ids = ["10001", "10343", "10285"]
for tid in template_ids:
macros = zabbix_api("usermacro.get", {
"output": ["hostmacroid", "macro", "value"],
"hostids": [tid]
}, auth=auth).get("result", [])
for m in macros:
mid = m["hostmacroid"]
macro_name = m["macro"]
if macro_name == "{$CPU.UTIL.CRIT}":
res = zabbix_api("usermacro.update", {"hostmacroid": mid, "value": "95"}, auth=auth)
print(f"Template {tid} macro {macro_name} updated to 95: {res.get('result')}")
elif macro_name == "{$MEMORY.UTIL.MAX}":
res = zabbix_api("usermacro.update", {"hostmacroid": mid, "value": "95"}, auth=auth)
print(f"Template {tid} macro {macro_name} updated to 95: {res.get('result')}")
elif macro_name == "{$LOAD_AVG_PER_CPU.MAX.WARN}":
res = zabbix_api("usermacro.update", {"hostmacroid": mid, "value": "2.5"}, auth=auth)
print(f"Template {tid} macro {macro_name} updated to 2.5: {res.get('result')}")
# 2. Update trigger expressions from 5m to 15m
# Triggers: 22370 (Passive CPU), 22376 (Passive Mem), 22415 (Active CPU), 22421 (Active Mem)
# In Zabbix 6.0/7.0, expression format is min(/<template_name>/<item_key>,15m)>{$MACRO}
# Let's inspect the exact expression syntax for trigger update
for tr_id in ["22370", "22376", "22415", "22421"]:
tr = zabbix_api("trigger.get", {
"output": ["triggerid", "description", "expression"],
"triggerids": [tr_id]
}, auth=auth).get("result", [{}])[0]
expr = tr.get("expression", "")
print(f"\nTrigger {tr_id} ({tr.get('description')}): current expr = {expr}")
# Replace 5m with 15m in the expression
if "5m" in expr:
new_expr = expr.replace("5m", "15m")
# In trigger.update, Zabbix expects expression with host/item syntax if updated
# Or description update: "Linux: High CPU utilization (>95% for 15m)"
desc = tr.get("description", "")
# Update trigger
up_res = zabbix_api("trigger.update", {
"triggerid": tr_id,
"expression": new_expr
}, auth=auth)
print(f" Updated expr to 15m: {up_res}")
print("\nFinished updating Zabbix noise reduction thresholds!")

65
scripts/apply_36.py Normal file
View File

@@ -0,0 +1,65 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
cmd = """
k3s kubectl -n dns delete svc pihole-dns-vip pihole-web-vip --wait=false
sleep 2
cat << 'EOF' | k3s kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: pihole-dns-vip
namespace: dns
annotations:
metallb.universe.tf/loadBalancerIPs: 140.44.4.36
metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
labels:
app: pihole
spec:
type: LoadBalancer
selector:
app: pihole
ports:
- name: dns-udp
port: 53
targetPort: 53
protocol: UDP
- name: dns-tcp
port: 53
targetPort: 53
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: pihole-web-vip
namespace: dns
annotations:
metallb.universe.tf/loadBalancerIPs: 140.44.4.36
metallb.universe.tf/allow-shared-ip: "aerodrome-dns-key"
labels:
app: pihole
spec:
type: LoadBalancer
selector:
app: pihole
ports:
- name: web-http
port: 80
targetPort: 80
protocol: TCP
EOF
sleep 3
k3s kubectl -n dns get svc
"""
stdin, stdout, stderr = c.exec_command(cmd)
print(stdout.read().decode('utf-8'))
c.close()

View File

@@ -0,0 +1,52 @@
import paramiko
import time
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
def apply_and_update(ip, name, manifest_path):
print(f"\n=======================================================")
print(f"Applying to {name} ({ip})...")
with open(manifest_path, 'r', encoding='utf-8') as f:
content = f.read()
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=key_path, timeout=10)
stdin, stdout, stderr = ssh.exec_command("k3s kubectl apply -f -")
stdin.write(content)
stdin.channel.shutdown_write()
print("Apply output:", stdout.read().decode())
err = stderr.read().decode()
if err:
print("Apply stderr:", err)
print(f"Waiting for Pi-hole deployment rollout on {name}...")
stdin, stdout, stderr = ssh.exec_command("k3s kubectl rollout status deployment/pihole -n dns --timeout=60s")
print(stdout.read().decode())
# Get new pod name
stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n dns -l app=pihole -o jsonpath='{.items[0].metadata.name}'")
pod_name = stdout.read().decode().strip()
print(f"New Pi-hole pod: {pod_name}")
# Set password to snarfnetadmin
print("Setting Pi-hole password to 'snarfnetadmin'...")
stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n dns {pod_name} -- pihole setpassword snarfnetadmin")
out = stdout.read().decode()
err = stderr.read().decode()
print("Set password output:", out)
if err:
print("Set password stderr:", err)
ssh.close()
aero_manifest = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml"
space_manifest = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml"
apply_and_update("140.44.4.11", "Aerodrome", aero_manifest)
apply_and_update("140.44.4.41", "Spaceport", space_manifest)
print("\nDone!")

View File

@@ -0,0 +1,51 @@
import paramiko
import time
targets = [
{
"name": "Node 1 (140.44.44.121)",
"ip": "140.44.44.121",
"settings": [
("bios.IntegratedDevices.SriovGlobalEnable", "Enabled")
]
},
{
"name": "Node 4 (140.44.44.124)",
"ip": "140.44.44.124",
"settings": [
("bios.SysProfileSettings.SysProfile", "PerfOptimized")
]
}
]
for t in targets:
print(f"\n==========================================")
print(f"Applying BIOS settings to {t['name']}...")
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(t['ip'], username='root', password='Katzir476!', timeout=15)
for key, val in t['settings']:
cmd = f"racadm set {key} {val}"
stdin, stdout, stderr = c.exec_command(cmd)
out = stdout.read().decode().strip()
print(f" racadm set {key} {val}:\n {out}")
time.sleep(1)
# Create BIOS job
stdin, stdout, stderr = c.exec_command("racadm jobqueue create BIOS.Setup.1-1")
job_out = stdout.read().decode().strip()
print(f" racadm jobqueue create BIOS.Setup.1-1:\n {job_out}")
# View job queue
stdin, stdout, stderr = c.exec_command("racadm jobqueue view")
q_out = stdout.read().decode().strip()
print(f" Job Queue:\n {q_out[:200]}")
c.close()
except Exception as e:
print(f" Error on {t['name']}: {e}")
print("\nDone applying BIOS settings!")

37
scripts/apply_clean_cm.py Normal file
View File

@@ -0,0 +1,37 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r', encoding='utf-8') as f:
text = f.read()
# Extract just the ConfigMap section
# It starts with "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: prometheus-config" and ends with "---"
parts = text.split("---")
cm_part = None
for p in parts:
if "name: prometheus-config" in p:
cm_part = p
break
if cm_part:
print("Found prometheus-config ConfigMap! Size:", len(cm_part))
stdin, stdout, stderr = c.exec_command("cat > /tmp/prom_cm_clean.yaml")
stdin.write(cm_part)
stdin.close()
stdout.channel.recv_exit_status()
_, o, e = c.exec_command("kubectl apply -f /tmp/prom_cm_clean.yaml")
print("Apply result:\n", o.read().decode().strip())
err = e.read().decode().strip()
if err:
print("Apply err:\n", err)
# Reload prometheus via lifecycle endpoint
_, o, _ = c.exec_command("kubectl exec deploy/prometheus -- wget -qO- --post-action='' http://localhost:9090/-/reload")
print("Reload endpoint output:", o.read().decode().strip())
c.close()

View File

@@ -0,0 +1,48 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
cm_yaml = """cat << 'EOF' | k3s kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns-custom
namespace: kube-system
data:
snarfnet.override: |
hosts {
140.44.4.34 aerodrome.snarfnet.net
140.44.4.90 harvester.spacedock.snarfnet.net
fallthrough
}
EOF
"""
print("Applying coredns-custom on Spaceport...")
stdin, stdout, stderr = client.exec_command(cm_yaml)
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
print("\nRestarting coredns deployment on Spaceport...")
client.exec_command("k3s kubectl -n kube-system rollout restart deployment/coredns")
client.exec_command("k3s kubectl -n kube-system rollout status deployment/coredns --timeout=60s")
print("\nRestarting cattle-cluster-agent on Spaceport...")
client.exec_command("k3s kubectl -n cattle-system rollout restart deployment/cattle-cluster-agent")
client.exec_command("k3s kubectl -n cattle-system rollout status deployment/cattle-cluster-agent --timeout=60s")
time.sleep(5)
print("\nChecking cattle-cluster-agent logs:")
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent --tail=25")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,31 @@
import paramiko
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
def apply_manifest(host, local_file):
with open(local_file, "r") as f:
manifest_content = f.read()
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, username="root", key_filename=SSH_KEY, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl apply -f -")
stdin.write(manifest_content)
stdin.close()
out = stdout.read().decode()
err = stderr.read().decode()
client.close()
return out, err
print("=== Applying to Aerodrome (140.44.4.11) ===")
out, err = apply_manifest("140.44.4.11", r"homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml")
print(out)
if err:
print("Stderr:", err)
print("=== Applying to Spaceport (140.44.4.41) ===")
out, err = apply_manifest("140.44.4.41", r"homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml")
print(out)
if err:
print("Stderr:", err)

View File

@@ -0,0 +1,250 @@
import yaml
def str_presenter(dumper, data):
if '\n' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.representer.SafeRepresenter.add_representer(str, str_presenter)
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
text = f.read()
docs = list(yaml.safe_load_all(text))
cm = docs[2]
services_yaml = """- Critical Services:
- Authentik:
icon: authentik.png
href: https://authentik.snarfnet.net
description: SSO & Identity Management
- PeaNUT UPS:
icon: peanut.png
href: https://portal.snarfnet.net
description: UPS Power & Battery Monitor
- Unifi Controller:
icon: unifi.png
href: https://unifi.ui.com
widget:
type: unifi
url: https://140.44.44.1
username: Snarf
password: Katzir476!!!
- GitOps & Automation:
- Gitea:
icon: gitea.png
href: https://git.snarfnet.net
description: Git Repositories
widget:
type: gitea
url: https://git.snarfnet.net
key: f19e772cf54e40ceb2befe452e2bc24dd81fdb71
- ArgoCD:
icon: argocd.png
href: https://argo.snarfnet.net
description: Declarative GitOps Reconciler
widget:
type: argocd
url: https://argo.snarfnet.net
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0
- AWX Automation:
icon: awx.png
href: https://awx.snarfnet.net
description: Ansible Automation Controller
- Semaphore:
icon: semaphore.png
href: https://semaphore.snarfnet.net
description: Playbook Execution Engine
- Storage & Clusters:
- Longhorn Aerodrome:
icon: longhorn.png
href: https://longhorn.aerodrome.snarfnet.net
description: Aerodrome Distributed Flash Storage
- Longhorn Spaceport:
icon: longhorn.png
href: https://longhorn.spaceport.snarfnet.net
description: Spaceport Distributed Block Storage
- Rancher (Aerodrome):
icon: rancher.png
href: https://aerodrome.snarfnet.net
description: Kubernetes Management Console
widget:
type: customapi
url: https://aerodrome.snarfnet.net/v3/clusters/local
headers:
Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
mappings:
- field: capacity.cpu
label: CPU Cores
- field: capacity.pods
label: Max Pods
- field: state
label: Cluster State
- Harvester HCI (Spacedock):
icon: harvester.png
href: https://harvester.spacedock.snarfnet.net
description: HCI Virtualization Hypervisors
widget:
type: customapi
url: https://aerodrome.snarfnet.net/v3/clusters/c-cclnt
headers:
Authorization: "Bearer kubeconfig-u-aqzkufonck4hmbd:bw22b96nv8844k8w5r8lzpz8gnk22tjwj95j6nkzr7hd5dhm4bs6z7"
mappings:
- field: capacity.cpu
label: CPU Cores
- field: capacity.pods
label: Max Pods
- field: state
label: HCI State
- Portainer Swarm:
icon: portainer.png
href: https://shipyard.snarfnet.net
description: Docker Swarm Container Fleet
widget:
type: portainer
url: https://140.44.4.71:9443
env: 3
key: ptr_TCVm8ujeB4zT0oRejyMCwKw5enaPG/uKj1KuF0uOWX8=
- Infrastructure & Observability:
- NGINX Proxy Manager:
icon: nginx-proxy-manager
href: https://nginx.snarfnet.net
description: Reverse Proxy & SSL Certificates
- Uptime Kuma:
icon: uptime-kuma
href: https://uptime.snarfnet.net
description: Multi-Endpoint Availability Probes
widget:
type: uptimekuma
url: http://140.44.4.28:448
slug: default
- Zabbix:
icon: zabbix.png
href: https://zabbix.snarfnet.net
description: Infrastructure Metrics & Alarms
- Grafana:
icon: grafana.png
href: https://grafana.snarfnet.net
description: Real-Time Metrics & AI Dashboards
widget:
type: grafana
url: https://grafana.snarfnet.net
username: admin
password: Katzir476!!!
- Core Node Glances:
icon: glances.png
href: http://140.44.4.11:61208
description: Host Resource Monitor
- Environment & Weather:
- Tempest Weather Station:
icon: weather.png
href: https://snarfhome.snarfnet.net
description: Backyard Weather Telemetry
widget:
type: homeassistant
url: http://14.4.4.20:8123
key: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIzZWUyNTE1MDk0NGU0ZjRjYmJkMGM1MWI5MThmY2EzYiIsImlhdCI6MTc0NjgxMDM3NSwiZXhwIjoyMDYyMTcwMzc1fQ.Le6M7WL63nxtLTxccFacQR5GVUY5C-a_zGtmXT_uaz4
custom:
- template: "{{ states('sensor.st_00158555_temperature') | float(0) | round(1) }} °F"
label: Outdoor Temp
- template: "{{ states('sensor.st_00158555_feels_like') | float(0) | round(1) }} °F"
label: Feels Like
- template: "{{ states('sensor.st_00158555_humidity') | float(0) | round(1) }} %"
label: Humidity
- template: "{{ states('sensor.st_00158555_wind_speed') | float(0) | round(1) }} mph"
label: Wind Speed
"""
settings_yaml = """title: SnarfNet Operations & Telemetry
headerStyle: boxed
color: slate
theme: dark
statusStyle: badge
background:
image: https://i.imgur.com/hOzzkCY.jpeg
blur: sm
opacity: 50
brightness: 50
cardBlur: sm
quicklaunch:
searchDescriptions: true
hideInternetSearch: true
hideVisitURL: true
hideVersion: true
layout:
Critical Services:
icon: mdi-shield-check
style: row
columns: 3
GitOps & Automation:
icon: mdi-git
style: row
columns: 4
Storage & Clusters:
icon: mdi-server-network
style: row
columns: 3
Infrastructure & Observability:
icon: mdi-chart-timeline-variant
style: row
columns: 3
Environment & Weather:
icon: mdi-weather-partly-cloudy
style: row
columns: 1
showStats: true
providers:
longhorn:
url: http://longhorn-backend.longhorn-system.svc.cluster.local:9500
"""
widgets_yaml = """- kubernetes:
cluster:
show: true
cpu: true
memory: true
show_names: true
nodes:
show: false
- longhorn:
total: true
nodes: false
- resources:
label: System
backend: resources
cpu: true
expanded: true
memory: true
disk: /
- search:
provider: duckduckgo
target: _blank
"""
bookmarks_yaml = """- Quick Links:
- GitHub:
- abbr: GH
href: https://github.com/
- UniFi Site Manager:
- abbr: UI
href: https://unifi.ui.com/
- Cloudflare:
- abbr: CF
href: https://dash.cloudflare.com/
"""
cm['data']['services.yaml'] = services_yaml
cm['data']['settings.yaml'] = settings_yaml
cm['data']['widgets.yaml'] = widgets_yaml
cm['data']['bookmarks.yaml'] = bookmarks_yaml
cm['data']['kubernetes.yaml'] = "mode: cluster\n"
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'w', encoding='utf-8') as f:
yaml.dump_all(docs, f, sort_keys=False, default_flow_style=False, allow_unicode=True)
print("Saved clean homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml successfully!")

View File

@@ -0,0 +1,33 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r", encoding="utf-8") as f:
grafana_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_updated.yaml")
stdin.write(grafana_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("Applying Grafana manifests:")
print(run("kubectl apply -f /tmp/grafana_updated.yaml"))
print("\nRestarting Grafana:")
print(run("kubectl rollout restart deployment/grafana -n default"))
print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
print("\nWaiting for Grafana to finish initialization...")
time.sleep(10)
c.close()

View File

@@ -0,0 +1,20 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
print("Applying server-side:")
print(run("kubectl apply -f /tmp/grafana_full.yaml --server-side --force-conflicts"))
print("\nRollout restart:")
print(run("kubectl rollout restart deployment/grafana -n default"))
print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
c.close()

View File

@@ -0,0 +1,149 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
adv_config = """# Buffer tuning for Authentik headers
proxy_buffers 8 16k;
proxy_buffer_size 32k;
# Outpost verification route - plain HTTP to internal VIP
location /outpost.goauthentik.io {
auth_request off;
proxy_pass http://140.44.4.29:9000/outpost.goauthentik.io;
proxy_set_header Host authentik.snarfnet.net;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
add_header Set-Cookie $auth_cookie;
auth_request_set $auth_cookie $upstream_http_set_cookie;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
}
# Redirect to Authentik when unauthenticated
location @goauthentik_proxy_signin {
internal;
auth_request off;
add_header Set-Cookie $auth_cookie;
return 302 https://authentik.snarfnet.net/outpost.goauthentik.io/start?rd=$scheme://$http_host$request_uri;
}
# Protected Topology HTML (Interactive)
location = /topology.html {
auth_request /outpost.goauthentik.io/auth/nginx;
error_page 401 = @goauthentik_proxy_signin;
auth_request_set $auth_cookie $upstream_http_set_cookie;
add_header Set-Cookie $auth_cookie;
add_header Cache-Control "no-cache, no-store, must-revalidate";
expires 0;
alias /data/topology/snarfnet_homelab_topology.html;
default_type text/html;
}
# Protected Topology PNG (High-Res Image)
location = /topology.png {
auth_request /outpost.goauthentik.io/auth/nginx;
error_page 401 = @goauthentik_proxy_signin;
auth_request_set $auth_cookie $upstream_http_set_cookie;
add_header Set-Cookie $auth_cookie;
add_header Cache-Control "no-cache, no-store, must-revalidate";
expires 0;
alias /data/topology/snarfnet_homelab_topology.png;
default_type image/png;
}
location = /snarfnet_homelab_topology.html {
auth_request /outpost.goauthentik.io/auth/nginx;
error_page 401 = @goauthentik_proxy_signin;
auth_request_set $auth_cookie $upstream_http_set_cookie;
add_header Set-Cookie $auth_cookie;
add_header Cache-Control "no-cache, no-store, must-revalidate";
expires 0;
alias /data/topology/snarfnet_homelab_topology.html;
default_type text/html;
}
location = /snarfnet_homelab_topology.png {
auth_request /outpost.goauthentik.io/auth/nginx;
error_page 401 = @goauthentik_proxy_signin;
auth_request_set $auth_cookie $upstream_http_set_cookie;
add_header Set-Cookie $auth_cookie;
add_header Cache-Control "no-cache, no-store, must-revalidate";
expires 0;
alias /data/topology/snarfnet_homelab_topology.png;
default_type image/png;
}
"""
stdin, stdout, stderr = c.exec_command("cat > /tmp/adv_config_h21.txt")
stdin.write(adv_config)
stdin.close()
stdout.read()
remote_script = """import sqlite3, shutil, os
db_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'
conf_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/21.conf'
with open('/tmp/adv_config_h21.txt', 'r') as f:
adv = f.read()
# Update DB
conn = sqlite3.connect(db_path)
cur = conn.cursor()
cur.execute("UPDATE proxy_host SET advanced_config = ? WHERE id = 21", (adv,))
conn.commit()
conn.close()
print("Updated database for host 21.")
# Update 21.conf
with open(conf_path, 'r') as f:
conf = f.read()
# Strip existing adv_config block if present
tag = '# Buffer tuning for Authentik headers'
if tag in conf:
idx_start = conf.find(tag)
idx_end = conf.find('location / {', idx_start)
if idx_start != -1 and idx_end != -1:
new_conf = conf[:idx_start] + adv + "\\n\\n " + conf[idx_end:]
else:
new_conf = conf
else:
idx = conf.find('location / {')
if idx != -1:
new_conf = conf[:idx] + adv + "\\n\\n " + conf[idx:]
else:
new_conf = conf.rstrip().rstrip('}') + "\\n\\n" + adv + "\\n}\\n"
with open(conf_path, 'w') as f:
f.write(new_conf)
print("Updated 21.conf.")
"""
stdin, stdout, stderr = c.exec_command("cat > /tmp/apply_h21.py")
stdin.write(remote_script)
stdin.close()
stdout.read()
stdin, stdout, stderr = c.exec_command("python3 /tmp/apply_h21.py")
print("Remote python stdout:", stdout.read().decode())
print("Remote python stderr:", stderr.read().decode())
stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -t")
out = stdout.read().decode()
err = stderr.read().decode()
print("nginx -t stdout:", out)
print("nginx -t stderr:", err)
if "successful" in out or "successful" in err:
print("Nginx syntax OK! Reloading nginx...")
stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q --filter 'name=nginx-proxy-manager_app') nginx -s reload")
print("Reload stdout:", stdout.read().decode())
print("Reload stderr:", stderr.read().decode())
c.close()

31
scripts/apply_homepage.py Normal file
View File

@@ -0,0 +1,31 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"\n>>> {cmd}")
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
if out:
print(out)
if err:
print("[stderr]", err)
return out
with open('homelab-gitops-repo/manifests/spaceport/homepage/homepage.yaml', 'r', encoding='utf-8') as f:
content = f.read()
stdin, stdout, stderr = c.exec_command('cat > /tmp/homepage.yaml')
stdin.write(content)
stdin.close()
stdout.read()
run("ls -lh /tmp/homepage.yaml")
run("kubectl apply -f /tmp/homepage.yaml")
run("kubectl rollout restart deployment/homepage-2-1744068158 -n infrastructure")
run("kubectl rollout status deployment/homepage-2-1744068158 -n infrastructure --timeout=60s")
c.close()

View File

@@ -0,0 +1,79 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run_sql(sql):
pod_cmd = "kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'"
_, o, _ = c.exec_command(pod_cmd)
pod = o.read().decode().strip()
cmd = f"kubectl exec -i -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root"
stdin, stdout, stderr = c.exec_command(cmd)
stdin.write(sql)
stdin.close()
return stdout.read().decode().strip(), stderr.read().decode().strip()
# Target monitors:
# 98: Spacedock Harvester Node 01
# 99: Spacedock Harvester Node 02
# 100: Spacedock Harvester Node 03
# 101: Spacedock Harvester Node 04
# 102: Dell PowerEdge R730 Node 1 iDRAC
# 103: Dell PowerEdge R730 Node 2 iDRAC
# 104: Dell PowerEdge R730 Node 3 iDRAC
# 105: Dell PowerEdge R730 Node 4 iDRAC
# Tags to attach:
# 5: ai
# 9: IP
# 11: Tier-2-Operational
monitors = [98, 99, 100, 101, 102, 103, 104, 105]
tag_ids = [5, 9, 11]
sql_commands = ["USE kuma;"]
for m_id in monitors:
for t_id in tag_ids:
# Check if already exists
check = f"SELECT id FROM monitor_tag WHERE monitor_id={m_id} AND tag_id={t_id};"
sql_commands.append(f"""
INSERT INTO monitor_tag (monitor_id, tag_id, value)
SELECT {m_id}, {t_id}, NULL
WHERE NOT EXISTS (SELECT 1 FROM monitor_tag WHERE monitor_id={m_id} AND tag_id={t_id});
""")
# Also ensure Harvester Console (57) and Harvester HCI Console VIP (94) have ai (5)
for m_id in [57, 94]:
sql_commands.append(f"""
INSERT INTO monitor_tag (monitor_id, tag_id, value)
SELECT {m_id}, 5, NULL
WHERE NOT EXISTS (SELECT 1 FROM monitor_tag WHERE monitor_id={m_id} AND tag_id=5);
""")
full_sql = "\n".join(sql_commands)
out, err = run_sql(full_sql)
if err:
print("SQL Error:", err)
else:
print("Tags applied successfully!")
# Verification query
verify_sql = """
USE kuma;
SELECT m.id, m.name, GROUP_CONCAT(t.name ORDER BY t.id SEPARATOR ', ') AS tags
FROM monitor m
JOIN monitor_tag mt ON m.id = mt.monitor_id
JOIN tag t ON mt.tag_id = t.id
WHERE m.id IN (57, 94, 98, 99, 100, 101, 102, 103, 104, 105)
GROUP BY m.id, m.name
ORDER BY m.id;
"""
res, _ = run_sql(verify_sql)
print("\nUpdated Monitors and Tags:\n", res)
c.close()

140
scripts/apply_npm_fixes.py Normal file
View File

@@ -0,0 +1,140 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
print("1. Creating backup of database and configs...")
backup_cmd = """
cp /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite.bak.$(date +%s)
"""
stdin, stdout, stderr = c.exec_command(backup_cmd)
stdout.channel.recv_exit_status()
print("Backup complete.")
print("2. Updating database.sqlite for hosts 45, 41, and 37...")
py_db = """
import sqlite3, datetime
conn = sqlite3.connect("/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite")
cur = conn.cursor()
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
updates = [
(45, "140.44.4.34", 443, "https"), # Longhorn -> Aerodrome Traefik
(41, "140.44.4.62", 443, "https"), # Semaphore -> Spaceport Traefik
(37, "140.44.4.34", 443, "https"), # Uptime Kuma -> Aerodrome Traefik
]
for pid, host, port, scheme in updates:
cur.execute('''UPDATE proxy_host
SET forward_host = ?, forward_port = ?, forward_scheme = ?,
allow_websocket_upgrade = 1, http2_support = 1, modified_on = ?
WHERE id = ?''', (host, port, scheme, now, pid))
print(f"DB updated: Host {pid} -> {scheme}://{host}:{port}")
conn.commit()
conn.close()
"""
stdin, stdout, stderr = c.exec_command(f"python3 -c '{py_db}'")
print(stdout.read().decode())
err = stderr.read().decode()
if err: print("DB err:", err)
print("3. Generating updated nginx configuration files...")
def generate_conf(pid, domains, cert_id, host, port, scheme):
return f"""# ------------------------------------------------------------
# {domains}
# ------------------------------------------------------------
map $scheme $hsts_header {{
https "max-age=63072000; preload";
}}
server {{
set $forward_scheme {scheme};
set $server "{host}";
set $port {port};
listen 80;
listen [::]:80;
listen 443 ssl;
listen [::]:443 ssl;
server_name {domains};
http2 on;
# Let's Encrypt SSL
include conf.d/include/letsencrypt-acme-challenge.conf;
include conf.d/include/ssl-cache.conf;
include conf.d/include/ssl-ciphers.conf;
ssl_certificate /etc/letsencrypt/live/npm-{cert_id}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/npm-{cert_id}/privkey.pem;
# Asset Caching
include conf.d/include/assets.conf;
# Block Exploits
include conf.d/include/block-exploits.conf;
# Force SSL
set $trust_forwarded_proto "F";
include conf.d/include/force-ssl.conf;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_http_version 1.1;
access_log /data/logs/proxy-host-{pid}_access.log proxy;
error_log /data/logs/proxy-host-{pid}_error.log warn;
location / {{
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_http_version 1.1;
# Proxy!
include conf.d/include/proxy.conf;
}}
# Custom
include /data/nginx/custom/server_proxy[.]conf;
}}
"""
configs = [
(45, "longhorn.aerodrome.snarfnet.net", 37, "140.44.4.34", 443, "https"),
(41, "semaphore.snarfnet.net", 33, "140.44.4.62", 443, "https"),
(37, "uptime.snarfnet.net", 27, "140.44.4.34", 443, "https"),
]
for pid, dom, cert_id, host, port, scheme in configs:
cfg = generate_conf(pid, dom, cert_id, host, port, scheme)
path = f"/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/{pid}.conf"
cmd = f"cat << 'EOF' > {path}\n{cfg}\nEOF\n"
stdin, stdout, stderr = c.exec_command(cmd)
stdout.channel.recv_exit_status()
print(f"Wrote config: {path}")
print("4. Testing Nginx config inside container...")
stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -t")
out = stdout.read().decode()
err = stderr.read().decode()
print("nginx -t stdout:", out)
print("nginx -t stderr:", err)
if "successful" in out or "successful" in err:
print("5. Reloading Nginx via Swarm service update...")
stdin, stdout, stderr = c.exec_command("docker service update --force nginx-proxy-manager_app")
print(stdout.read().decode())
print("Reload completed.")
else:
print("ERROR: Nginx config test failed!")
c.close()

View File

@@ -0,0 +1,26 @@
import paramiko
def apply_manifest(host, cluster_name, file_path):
print(f"Applying {file_path} to {cluster_name} ({host})...")
with open(file_path, 'r') as f:
manifest_content = f.read()
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('k3s kubectl apply -f -')
stdin.write(manifest_content)
stdin.channel.shutdown_write()
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
print("Output:")
print(out)
if err:
print("Stderr:", err)
ssh.close()
apply_manifest('140.44.4.11', 'Aerodrome', r'homelab-gitops-repo/manifests/aerodrome/dns/dns-stack.yaml')
apply_manifest('140.44.4.41', 'Spaceport', r'homelab-gitops-repo/manifests/spaceport/dns/dns-stack.yaml')

View File

@@ -0,0 +1,38 @@
import paramiko
import yaml
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r') as f:
docs = list(yaml.safe_load_all(f))
# Find ConfigMap prometheus-config
cm_doc = None
for d in docs:
if d and d.get('kind') == 'ConfigMap' and d.get('metadata', {}).get('name') == 'prometheus-config':
cm_doc = d
break
if cm_doc:
cm_yaml = yaml.dump(cm_doc)
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_cm.yaml")
stdin.write(cm_yaml)
stdin.close()
stdout.channel.recv_exit_status()
_, o, e = c.exec_command("kubectl apply -f /tmp/prometheus_cm.yaml")
print("CM apply:\n", o.read().decode().strip())
err = e.read().decode().strip()
if err:
print("CM err:", err)
# Restart prometheus pod
_, o, _ = c.exec_command("kubectl rollout restart deployment/prometheus")
print("Prometheus restart:", o.read().decode().strip())
_, o, _ = c.exec_command("kubectl rollout status deployment/prometheus --timeout=60s")
print("Rollout status:", o.read().decode().strip())
c.close()

View File

@@ -0,0 +1,45 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r") as f:
prom_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml")
stdin.write(prom_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("Applying Prometheus:")
print(run("kubectl apply -f /tmp/prometheus.yaml"))
with open("homelab-gitops-repo/manifests/aerodrome/grafana/grafana.yaml", "r") as f:
grafana_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/grafana_full.yaml")
stdin.write(grafana_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("\nApplying Grafana configuration & dashboards:")
print(run("kubectl apply -f /tmp/grafana_full.yaml"))
print("\nRestarting Grafana:")
print(run("kubectl rollout restart deployment/grafana -n default"))
print(run("kubectl rollout status deployment/grafana -n default --timeout=60s"))
print("\nChecking Prometheus pod:")
time.sleep(5)
print(run("kubectl get pods -l app.kubernetes.io/name=prometheus"))
c.close()

View File

@@ -0,0 +1,36 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r') as f:
prom_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml")
stdin.write(prom_yaml)
stdin.close()
stdout.channel.recv_exit_status()
_, o, e = c.exec_command("kubectl apply -f /tmp/prometheus.yaml")
print("Kubectl apply:\n", o.read().decode().strip())
err = e.read().decode().strip()
if err:
print("Err:", err)
# Find prometheus pod
_, o, _ = c.exec_command("kubectl get pod -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}'")
pod = o.read().decode().strip()
print("Prometheus pod:", pod)
# Reload config
_, o, _ = c.exec_command(f"kubectl exec {pod} -- wget -qO- --post-action='' http://localhost:9090/-/reload || kubectl exec {pod} -- curl -X POST http://localhost:9090/-/reload")
print("Reload result:", o.read().decode().strip())
time.sleep(5)
# Check targets for shipyard
_, o, _ = c.exec_command(f"""kubectl exec {pod} -- wget -qO- 'http://localhost:9090/api/v1/targets' | grep -o '"cluster":"shipyard"[^}}]*' | head -10""")
print("Shipyard targets found in Prometheus:\n", o.read().decode().strip())
c.close()

View File

@@ -0,0 +1,36 @@
import paramiko
import time
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
out = o.read().decode().strip()
err = e.read().decode().strip()
return out + ("\nERR: " + err if err else "")
with open("homelab-gitops-repo/manifests/aerodrome/prometheus/prometheus.yaml", "r", encoding="utf-8") as f:
prom_yaml = f.read()
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_updated.yaml")
stdin.write(prom_yaml)
stdin.close()
stdout.channel.recv_exit_status()
print("Applying Prometheus:")
print(run("kubectl apply -f /tmp/prometheus_updated.yaml"))
print("\nRestarting Prometheus:")
print(run("kubectl rollout restart deployment/prometheus -n default"))
print(run("kubectl rollout status deployment/prometheus -n default --timeout=60s"))
print("\nWaiting 15 seconds for scrape loop...")
time.sleep(15)
print("\nPrometheus active targets count:")
print(run("kubectl exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets | grep -o '\"health\":\"up\"' | wc -l"))
c.close()

View File

@@ -0,0 +1,72 @@
import paramiko
import urllib.request
import ssl
import json
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
print("1. Updating Authentik Provider 10 redirect_uris...")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
token = "SnarfNetPermanentAutomationToken2026"
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json"
}
provider_update = {
"redirect_uris": [
{"matching_mode": "strict", "url": "https://semaphore.snarfnet.net/auth/oidc/authentik/redirect"},
{"matching_mode": "strict", "url": "https://semaphore.snarfnet.net/api/auth/oidc/authentik/redirect/"},
{"matching_mode": "regex", "url": "https://semaphore\\.snarfnet\\.net/.*"}
]
}
req = urllib.request.Request(
"https://authentik.snarfnet.net/api/v3/providers/oauth2/10/",
headers=headers,
data=json.dumps(provider_update).encode('utf-8'),
method="PATCH"
)
try:
with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
res = json.loads(resp.read().decode())
print(" ✓ Authentik Provider 10 updated successfully:", res.get('redirect_uris'))
except Exception as e:
print(" ✗ Failed to update Authentik Provider 10:", e)
print("\n2. Applying updated Semaphore manifest to Spaceport (140.44.4.41)...")
manifest_path = r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\semaphore\semaphore.yaml"
with open(manifest_path, 'r', encoding='utf-8') as f:
content = f.read()
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = ssh.exec_command("k3s kubectl apply -f -")
stdin.write(content)
stdin.channel.shutdown_write()
print("Apply output:", stdout.read().decode())
err = stderr.read().decode()
if err: print("Apply stderr:", err)
print("3. Rolling out Semaphore deployment...")
stdin, stdout, stderr = ssh.exec_command("k3s kubectl rollout restart deployment/semaphore -n infrastructure; k3s kubectl rollout status deployment/semaphore -n infrastructure --timeout=90s")
print(stdout.read().decode())
# Check new pod env
stdin, stdout, stderr = ssh.exec_command("k3s kubectl get pods -n infrastructure -l app.kubernetes.io/name=semaphore -o jsonpath='{.items[0].metadata.name}'")
new_pod = stdout.read().decode().strip()
print(f"New pod: {new_pod}")
stdin, stdout, stderr = ssh.exec_command(f"k3s kubectl exec -n infrastructure {new_pod} -- env | grep SEMAPHORE_OIDC_PROVIDERS")
print("New env:", stdout.read().decode())
ssh.close()
print("Done!")

View File

@@ -0,0 +1,161 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
print("1. Updating SQLite database for hosts 11 and 28...")
dsm_adv = """proxy_buffer_size 128k;
proxy_buffers 8 256k;
proxy_busy_buffers_size 256k;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_ssl_verify off;"""
py_db = f"""
import sqlite3, datetime
conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
cur = conn.cursor()
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
adv = '''{dsm_adv}'''
# Host 11: sn-nas.snarfnet.net -> 14.10.10.71:5001
cur.execute('''UPDATE proxy_host
SET forward_host = "14.10.10.71", forward_port = 5001, forward_scheme = "https",
allow_websocket_upgrade = 1, http2_support = 1, advanced_config = ?, modified_on = ?
WHERE id = 11''', (adv, now))
# Host 28: snarfnas.snarfnet.net -> 14.10.10.72:5001 (Clean DSM, remove dead outpost)
cur.execute('''UPDATE proxy_host
SET forward_host = "14.10.10.72", forward_port = 5001, forward_scheme = "https",
allow_websocket_upgrade = 1, http2_support = 1, advanced_config = ?, modified_on = ?
WHERE id = 28''', (adv, now))
conn.commit()
print("Database updated for hosts 11 and 28.")
conn.close()
"""
sftp = c.open_sftp()
with sftp.open('/tmp/update_nas_db.py', 'w') as f:
f.write(py_db)
sftp.close()
stdin, stdout, stderr = c.exec_command("python3 /tmp/update_nas_db.py")
print(stdout.read().decode())
err = stderr.read().decode()
if err: print("DB err:", err)
print("2. Generating Nginx configuration files for hosts 11 and 28...")
def make_dsm_conf(pid, domain, cert_id, host, port):
return f"""# ------------------------------------------------------------
# {domain}
# ------------------------------------------------------------
map $scheme $hsts_header {{
https "max-age=63072000;includeSubDomains; preload";
}}
server {{
set $forward_scheme https;
set $server "{host}";
set $port {port};
listen 80;
listen [::]:80;
listen 443 ssl;
listen [::]:443 ssl;
server_name {domain};
http2 on;
# Let's Encrypt SSL
include conf.d/include/letsencrypt-acme-challenge.conf;
include conf.d/include/ssl-cache.conf;
include conf.d/include/ssl-ciphers.conf;
ssl_certificate /etc/letsencrypt/live/npm-{cert_id}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/npm-{cert_id}/privkey.pem;
# Asset Caching
include conf.d/include/assets.conf;
# Block Exploits
include conf.d/include/block-exploits.conf;
# HSTS
add_header Strict-Transport-Security $hsts_header always;
# Force SSL
set $trust_forwarded_proto "F";
include conf.d/include/force-ssl.conf;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_http_version 1.1;
access_log /data/logs/proxy-host-{pid}_access.log proxy;
error_log /data/logs/proxy-host-{pid}_error.log warn;
location / {{
# HSTS
add_header Strict-Transport-Security $hsts_header always;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_http_version 1.1;
# DSM Buffer Tuning
proxy_buffer_size 128k;
proxy_buffers 8 256k;
proxy_busy_buffers_size 256k;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_ssl_verify off;
# Proxy!
include conf.d/include/proxy.conf;
}}
# Custom
include /data/nginx/custom/server_proxy[.]conf;
}}
"""
conf11 = make_dsm_conf(11, "sn-nas.snarfnet.net", 12, "14.10.10.71", 5001)
conf28 = make_dsm_conf(28, "snarfnas.snarfnet.net", 22, "14.10.10.72", 5001)
path11 = "/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/11.conf"
path28 = "/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/28.conf"
stdin, stdout, stderr = c.exec_command(f"cat << 'EOF' > {path11}\n{conf11}\nEOF\n")
stdout.channel.recv_exit_status()
print(f"Wrote {path11}")
stdin, stdout, stderr = c.exec_command(f"cat << 'EOF' > {path28}\n{conf28}\nEOF\n")
stdout.channel.recv_exit_status()
print(f"Wrote {path28}")
print("3. Testing Nginx configuration...")
stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -t")
out = stdout.read().decode()
err = stderr.read().decode()
print("nginx -t stdout:", out)
print("nginx -t stderr:", err)
if "successful" in out or "successful" in err:
print("4. Reloading Nginx Swarm service...")
stdin, stdout, stderr = c.exec_command("docker service update --force nginx-proxy-manager_app")
print(stdout.read().decode())
print("Reload complete!")
else:
print("ERROR: Nginx config invalid!")
c.close()

22
scripts/apply_web_vips.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
import sys
import base64
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
def apply_manifest(host, file_path):
with open(file_path, "rb") as f:
b64 = base64.b64encode(f.read()).decode('ascii')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(host, username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = c.exec_command(f"echo '{b64}' | base64 -d | k3s kubectl apply -f -")
print(f"Apply on {host}:")
print(stdout.read().decode('utf-8'))
c.close()
apply_manifest("140.44.4.11", r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\dns\dns-stack.yaml")
apply_manifest("140.44.4.41", r"C:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\spaceport\dns\dns-stack.yaml")

View File

@@ -0,0 +1,53 @@
import paramiko
import json
TARGET_VIPS = {
("monitoring", "uptime-lb"): "140.44.4.28",
("infra", "authentik-lb"): "140.44.4.29",
("awx", "awx-lb"): "140.44.4.30",
("infra", "technitium-npm-vip-service"): "140.44.4.31",
("default", "grafana-lb"): "140.44.4.32",
("infra", "unified-npm-vip-service"): "140.44.4.33",
("kube-system", "traefik"): "140.44.4.34",
("infra", "ak-outpost-authentik-ldap-outpost"): "140.44.4.35",
("longhorn-system", "longhorn-frontend-lb"): "140.44.4.36",
}
def main():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
print("=== PINNING AERODROME LOADBALANCER VIPS ===")
for (ns, name), target_ip in TARGET_VIPS.items():
patch = json.dumps({
"metadata": {
"annotations": {
"metallb.universe.tf/loadBalancerIP": target_ip
}
},
"spec": {
"loadBalancerIP": target_ip
}
})
cmd = f"kubectl patch svc {name} -n {ns} --type=merge -p '{patch}'"
stdin, stdout, stderr = ssh.exec_command(cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
print(f"[{ns}/{name}] -> {target_ip}: {out} {err}")
print("\n=== VERIFYING FINAL ALLOCATION STATE ===")
for (ns, name), target_ip in TARGET_VIPS.items():
stdin, stdout, stderr = ssh.exec_command(f"kubectl get svc {name} -n {ns} -o json")
svc = json.loads(stdout.read().decode())
current_ingress = [x.get('ip') for x in svc.get('status', {}).get('loadBalancer', {}).get('ingress', [])]
spec_ip = svc.get('spec', {}).get('loadBalancerIP')
ann_ip = svc.get('metadata', {}).get('annotations', {}).get('metallb.universe.tf/loadBalancerIP')
status = "LOCKED" if (spec_ip == target_ip and ann_ip == target_ip and target_ip in current_ingress) else "MISMATCH"
print(f"[{status}] {ns}/{name}: Ingress={current_ingress}, Spec={spec_ip}, Annotation={ann_ip}")
ssh.close()
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,30 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
def run_query(title, q):
cmd = f"""k3s kubectl -n infra exec authentik-postgresql-0 -- bash -c "PGPASSWORD=\\$(cat /opt/bitnami/postgresql/secrets/password) psql -U authentik -d authentik -c \\"{q}\\"" """
stdin, stdout, stderr = c.exec_command(cmd)
print(f"=== {title} ===")
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
if out.strip(): print(out.strip())
if err.strip(): print("ERR:", err.strip())
print()
run_query("OUTPOST PROVIDERS", "SELECT outpost_id, provider_id FROM authentik_outposts_outpost_providers;")
c.close()

View File

@@ -0,0 +1,21 @@
import paramiko
def inspect_cluster(ip, name):
print(f"\n=======================================================")
print(f"=== {name} WORKLOADS ({ip}) ===")
print(f"=======================================================")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "kubectl get deploy,sts,ds -A -o custom-columns=NAMESPACE:.metadata.namespace,KIND:.kind,NAME:.metadata.name,IMAGES:.spec.template.spec.containers[*].image"
_, out, _ = ssh.exec_command(cmd)
lines = out.read().decode().splitlines()
for l in lines:
if not any(k in l for k in ['kube-system', 'cattle-', 'metallb-system', 'longhorn-system']):
print(l)
ssh.close()
inspect_cluster('140.44.4.11', 'AERODROME')
inspect_cluster('140.44.4.41', 'SPACEPORT')

View File

@@ -0,0 +1,40 @@
import os
import re
import paramiko
manifest_root = r"c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests"
print("=== GITOPS MANIFEST IMAGE AUDIT ===")
image_re = re.compile(r'image:\s*([^\s"\']+)', re.IGNORECASE)
chart_re = re.compile(r'chart:\s*([^\s"\']+)', re.IGNORECASE)
version_re = re.compile(r'version:\s*([^\s"\']+)', re.IGNORECASE)
manifest_data = {}
for root, dirs, files in os.walk(manifest_root):
for f in files:
if f.endswith('.yaml') or f.endswith('.yml'):
path = os.path.join(root, f)
rel_path = os.path.relpath(path, manifest_root)
with open(path, 'r', encoding='utf-8', errors='ignore') as fp:
content = fp.read()
images = image_re.findall(content)
charts = chart_re.findall(content)
versions = version_re.findall(content)
if images or charts:
manifest_data[rel_path] = {
'images': list(set(images)),
'charts': list(set(charts)),
'versions': list(set(versions))
}
for rel_path, data in sorted(manifest_data.items()):
print(f"\nFile: {rel_path}")
if data['images']:
print(" Images:", ", ".join(data['images']))
if data['charts']:
print(" Charts:", ", ".join(data['charts']))
if data['versions']:
print(" Versions:", ", ".join(data['versions']))

View File

@@ -0,0 +1,75 @@
import paramiko
import time
import re
nodes = {
"Node 1 (snaihvs0lnx01)": "140.44.44.121",
"Node 2 (snaihvs0lnx02)": "140.44.44.122",
"Node 3 (snaihvs0lnx03)": "140.44.44.123",
"Node 4 (idrac-spacedock4)": "140.44.44.124",
}
results = {}
for name, ip in nodes.items():
print(f"Connecting to {name} ({ip})...")
node_data = {}
try:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(ip, username="root", password="Katzir476!", timeout=10)
chan = c.invoke_shell()
time.sleep(2)
if chan.recv_ready():
chan.recv(4096)
def send_and_recv(cmd, wait_time=2.5):
chan.send(cmd + "\r\n")
time.sleep(wait_time)
out = ""
while chan.recv_ready():
out += chan.recv(8192).decode('utf-8', errors='ignore')
time.sleep(0.5)
return out
# 1. BIOS Settings
raw_bios = send_and_recv("racadm get bios.SysProfileSettings", 2)
for key in ["SysProfile", "ProcCStates", "ProcC1E", "ProcTurboMode", "ProcPwrPerf"]:
m = re.search(rf"{key}\s*=\s*(\w+)", raw_bios)
node_data[key] = m.group(1) if m else "Unknown"
raw_proc = send_and_recv("racadm get bios.ProcSettings", 2)
for key in ["ProcVirtualization", "ProcExecuteDisable", "LogicalProc"]:
m = re.search(rf"{key}\s*=\s*(\w+)", raw_proc)
node_data[key] = m.group(1) if m else "Unknown"
raw_dev = send_and_recv("racadm get bios.IntegratedDevices", 2)
for key in ["SriovGlobalEnable"]:
m = re.search(rf"{key}\s*=\s*(\w+)", raw_dev)
node_data[key] = m.group(1) if m else "Unknown"
# 2. Storage / RAID
raw_ctrl = send_and_recv("racadm raid get controllers", 2)
controllers = [line.strip() for line in raw_ctrl.splitlines() if "RAID." in line or "PERC" in line or "AHCI." in line]
node_data["Controllers"] = ", ".join(controllers) if controllers else "None"
raw_vdisks = send_and_recv("racadm raid get vdisks", 3)
vdisks = [line.strip() for line in raw_vdisks.splitlines() if line.startswith("Disk.Virtual")]
node_data["VirtualDisks"] = len(vdisks)
node_data["VirtualDisksList"] = vdisks
raw_pdisks = send_and_recv("racadm raid get pdisks", 4)
pdisk_lines = [line.strip() for line in raw_pdisks.splitlines() if line.startswith("Disk.Bay")]
node_data["PhysicalDisks"] = len(pdisk_lines)
node_data["PDisksSummary"] = raw_pdisks
c.close()
except Exception as e:
node_data["Error"] = str(e)
results[name] = node_data
print("\n\n========================= SUMMARY MATRIX =========================")
import json
print(json.dumps(results, indent=2))

View File

@@ -0,0 +1,24 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.90', username='rancher', password='Katzir476!', timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(f"echo Katzir476! | sudo -S sh -c '{cmd}'")
return stdout.read().decode().strip()
print("=== CPU & VIRTUALIZATION (lscpu) ===")
print(run("lscpu | grep -E 'Model name|CPU\(s\):|Thread|Socket|Virtualization|NUMA'"))
print("\n=== KERNEL PARAMETERS (/proc/cmdline) ===")
print(run("cat /proc/cmdline"))
print("\n=== STORAGE CONTROLLER (lspci) ===")
print(run("lspci | grep -E 'RAID|SAS|SATA|Ethernet|Network'"))
print("\n=== BLOCK DEVICES (lsblk) ===")
print(run("lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS"))
c.close()

View File

@@ -0,0 +1,84 @@
import paramiko
import json
import time
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
HARVESTER_VIP = "140.44.4.90"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
print(f"Connecting to Harvester VIP {HARVESTER_VIP}...")
c.connect(HARVESTER_VIP, username="rancher", key_filename=SSH_KEY, timeout=10)
print("Connected to Harvester VIP!")
def run_k8s(cmd):
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
stdin, stdout, stderr = c.exec_command(full_cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
return out
print("\n=== 1. HARVESTER CLUSTER NODES ===")
print(run_k8s("kubectl get nodes -o wide"))
print("\n=== 2. NODE UPTIME ===")
print(run_k8s("uptime"))
print("\n=== 3. VIRTUAL MACHINES (KubeVirt) ===")
print(run_k8s("kubectl get vms,vmis -A -o wide"))
print("\n=== 4. HARVESTER SYSTEM POD STATUS (Not Running / Degraded) ===")
print(run_k8s("kubectl get pods -A | grep -v 'Running\\|Completed'"))
print("\n=== 5. LONGHORN STORAGE VOLUME STATUS ===")
print(run_k8s("kubectl get volumes.longhorn.io -n longhorn-system"))
c.close()
except Exception as e:
print(f"Error connecting to Harvester: {e}")
# Check iDRACs
idracs = {
"Spacedock Node 1": "140.44.44.121",
"Spacedock Node 2": "140.44.44.122",
"Spacedock Node 3": "140.44.44.123",
"Spacedock Node 4": "140.44.44.124",
}
print("\n=== 6. IDRAC SYSTEM EVENT LOGS & POWER STATUS ===")
for name, ip in idracs.items():
print(f"\n--- {name} ({ip}) ---")
try:
ic = paramiko.SSHClient()
ic.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ic.connect(ip, username="root", password="Katzir476!", timeout=8)
chan = ic.invoke_shell()
time.sleep(1.5)
if chan.recv_ready():
chan.recv(4096)
def racadm(cmd):
chan.send(cmd + "\r\n")
time.sleep(2)
out = ""
while chan.recv_ready():
out += chan.recv(8192).decode('utf-8', errors='ignore')
time.sleep(0.5)
return out
pwr = racadm("racadm serveraction powerstatus")
for line in pwr.splitlines():
if "power" in line.lower() or "server" in line.lower():
print(line.strip())
sel = racadm("racadm getsel -s 10")
print("Last 10 SEL Events:")
for line in sel.splitlines()[-12:]:
print(" ", line.strip())
ic.close()
except Exception as e:
print(f"Error querying {name}: {e}")

View File

@@ -0,0 +1,39 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
spaceport_nodes = [
('sninfr0lnxctl01', '140.44.4.41'),
('sninfr0lnxctl02', '140.44.4.42'),
('sninfr0lnxctl03', '140.44.4.43'),
('sninfr0lnxctl04', '140.44.4.44'),
('sninfr0lnxwrk01', '140.44.4.51'),
('sninfr0lnxwrk02', '140.44.4.52'),
('sninfr0lnxwrk03', '140.44.4.53'),
('sninfr0lnxwrk04', '140.44.4.54'),
('sninfr0lnxwrk05', '140.44.4.55'),
('sninfr0lnxwrk06', '140.44.4.56'),
]
print(f"{'NODE':<18} {'IP':<15} {'RUNNING KERNEL':<22} {'LATEST MODULES':<25} {'VETH LOADED':<12} {'UPTIME'}")
print("="*105)
for name, ip in spaceport_nodes:
try:
client.connect(ip, username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = client.exec_command("uname -r; ls -1 /lib/modules | sort -V | tail -n 1; lsmod | grep -q veth && echo YES || echo NO; uptime -p")
lines = [l.strip() for l in stdout.read().decode('utf-8').strip().split('\n') if l.strip()]
running_k = lines[0] if len(lines) > 0 else "?"
latest_mod = lines[1] if len(lines) > 1 else "?"
veth = lines[2] if len(lines) > 2 else "?"
uptime_str = lines[3] if len(lines) > 3 else "?"
print(f"{name:<18} {ip:<15} {running_k:<22} {latest_mod:<25} {veth:<12} {uptime_str}")
client.close()
except Exception as e:
print(f"{name:<18} {ip:<15} ERROR: {e}")

View File

@@ -0,0 +1,48 @@
import json, ssl, urllib.request
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
ZABBIX_URL = "http://140.44.4.60:8888/api_jsonrpc.php"
def zbx_req(method, params, token=None):
payload = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1}
headers = {'Content-Type': 'application/json-rpc'}
if token:
headers['Authorization'] = f'Bearer {token}'
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode('utf-8'), headers=headers, method='POST')
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
return json.loads(r.read().decode())['result']
token = zbx_req('user.login', {'username': 'Admin', 'password': 'zabbix'})
print("Zabbix Auth Token:", token)
# 1. Actions
actions = zbx_req('action.get', {'output': 'extend', 'selectOperations': 'extend', 'selectFilter': 'extend'}, token)
print("\n=== TRIGGER ACTIONS ===")
for a in actions:
print(f"ID: {a['actionid']}, Name: {a['name']}, Status (0=enabled, 1=disabled): {a['status']}, EventSource: {a['eventsource']}")
print(f" Operations: {a.get('operations')}")
print(f" Filter: {a.get('filter')}")
# 2. Media Types
media_types = zbx_req('mediatype.get', {'output': 'extend'}, token)
print("\n=== MEDIA TYPES ===")
for m in media_types:
print(f"ID: {m['mediatypeid']}, Name: {m['name']}, Type: {m['type']}, Status: {m['status']}")
# 3. User Media
users = zbx_req('user.get', {'output': ['userid', 'username'], 'selectMedias': 'extend'}, token)
print("\n=== USERS & MEDIA ===")
for u in users:
print(f"User: {u['username']} (ID: {u['userid']})")
for med in u.get('medias', []):
print(f" Media: mediatypeid={med['mediatypeid']}, sendto={med['sendto']}, active={med['active']}, severity={med['severity']}")
# 4. Recent Alerts
alerts = zbx_req('alert.get', {'output': 'extend', 'limit': 15, 'sortfield': 'clock', 'sortorder': 'DESC'}, token)
print("\n=== RECENT ALERTS ===")
for al in alerts:
print(f"Alert ID: {al['alertid']}, ActionID: {al['actionid']}, Clock: {al['clock']}, Error: {al.get('error')}, Status: {al.get('status')}, Subject: {al.get('subject')}")

View File

@@ -0,0 +1,35 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
pub_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6ycMke4AaV2qzWfULZCTZOKhSKf4pY2qRGYjYzs1MGKutJxt2jkYNK2aS0XffxLRR49v23zHbBwLYraNl2Mkb9rsKsyvKn8CAa1CEl9NxOci4LDRT8CeWHRnMZn8Mdu1pfeRwM26/Bjnd4RJrSKC7fyM0OlOpEp0DgnuPBqBtPMFGw+TCUT8eZ0pBrAI94wEan4l35kmuBcqm6RH3KuD1dDJKnPLzJ3/T21CViYL0GgFIDDor/hGI45VfOI/jKmkgM4YoudTCde6uPV5e2yaJpORZlR4nDxg9ntXp8HxbHGZkXYr55Ibsjem0lMU6nBh33jGbJVvat1Ay4oNCbygJsHfC/8ppe0bnLhnasxVAXhTvklZZL/sf7K/VS0ZifpfqB6DKtr1lziOm2wQeahv+Y1uoHkVNa3XY8Fx7xGPeT1BGd9lfcbUoweoTdMPMIPaSZlNGY32AHYkM4PShG9yI1coAqXzMby0+Z/Qwtr4l6cLKVvcGgqh5sd+z1+YvZbs= root@SNINFR2LNXCTL01"
# 1. Authorize on 140.44.4.71
c71 = paramiko.SSHClient()
c71.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c71.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
cmd = f"""
mkdir -p ~/.ssh && chmod 700 ~/.ssh
grep -qF '{pub_key}' ~/.ssh/authorized_keys 2>/dev/null || echo '{pub_key}' >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
"""
c71.exec_command(cmd)
c71.close()
print("Authorized 140.44.4.11 key on 140.44.4.71.")
# 2. Test SSH from 140.44.4.11 to 140.44.4.71
c11 = paramiko.SSHClient()
c11.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c11.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = c11.exec_command("ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@140.44.4.71 hostname")
out = stdout.read().decode('utf-8').strip()
err = stderr.read().decode('utf-8').strip()
print("Result of ssh from 140.44.4.11 to 140.44.4.71:")
print("STDOUT:", out)
if err: print("STDERR:", err)
c11.close()

View File

@@ -0,0 +1,467 @@
import json
import requests
import copy
import urllib3
urllib3.disable_warnings()
auth = ("admin", "Katzir476!!!")
base_url = "https://grafana.snarfnet.net"
# Load templates
with open("homelab-gitops-repo/dashboards/grafana/systems-dashboard.json") as f:
sys_template = json.load(f)
with open("homelab-gitops-repo/dashboards/grafana/container-metrics-dashboard.json") as f:
cnt_template = json.load(f)
def build_cluster_dashboard(cluster_key, cluster_title, folder_id, folder_uid, cluster_emoji):
dash = {
"annotations": {"list": []},
"editable": True,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": None,
"links": [],
"liveNow": False,
"refresh": "15s",
"schemaVersion": 39,
"tags": [cluster_key, "resource-monitor", "infrastructure"],
"time": {"from": "now-1h", "to": "now"},
"timepicker": {"refresh_intervals": ["5s", "10s", "15s", "30s", "1m", "5m"]},
"timezone": "browser",
"title": f"{cluster_emoji} {cluster_title} Resource Monitor",
"uid": f"{cluster_key}-resource-monitor",
"version": 1,
"panels": []
}
panels = []
current_y = 0
# ==========================================
# GROUP 1: Systems & Cluster Host Performance
# ==========================================
# Elevate Systems Row Header
panels.append({
"collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
"id": 100,
"title": f"🖥️ Systems & Host Performance — {cluster_title} Cluster Fleet",
"type": "row"
})
current_y += 1
# Systems Stat Panels (height 4)
stat_panels = [
{
"id": 1,
"title": f"Online Nodes ({cluster_title})",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'count(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Nodes", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 2,
"title": "Avg Cluster CPU Load (1m)",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'avg(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Load", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}, {"color": "orange", "value": 2.0}, {"color": "red", "value": 4.0}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 3,
"title": "Total Monitored RAM",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})', "legendFormat": "RAM", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 4,
"title": "Cluster RAM In-Use %",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'(1 - (sum(node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}) / sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}}))) * 100', "legendFormat": "In-Use", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
}
]
panels.extend(stat_panels)
current_y += 4
# Systems CPU Timeseries
panels.append({
"id": 5,
"title": "CPU Utilization by Host (%)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'100 - (avg by (instance) (rate(node_cpu_seconds_total{{mode="idle", cluster="{cluster_key}"}}[2m])) * 100)', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 6,
"title": "1-Minute Load Average by Host",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'node_load1{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none", "min": 0}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
current_y += 8
# Systems Memory Timeseries
panels.append({
"id": 7,
"title": "Memory Used % by Host",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'(1 - (node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}} / node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 8,
"title": "Available RAM by Host (GiB)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "bytes"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "min"]}}
})
current_y += 8
# Filesystem & Disk I/O
panels.append({
"id": 9,
"title": "Root (/) Filesystem Usage %",
"type": "bargauge",
"gridPos": {"h": 8, "w": 8, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'(1 - (node_filesystem_avail_bytes{{mountpoint="/", cluster="{cluster_key}"}} / node_filesystem_size_bytes{{mountpoint="/", cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent", "min": 0, "max": 100}},
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
})
panels.append({
"id": 10,
"title": "Disk Read Throughput",
"type": "timeseries",
"gridPos": {"h": 8, "w": 8, "x": 8, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_disk_read_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 11,
"title": "Disk Write Throughput",
"type": "timeseries",
"gridPos": {"h": 8, "w": 8, "x": 16, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_disk_written_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
# Network RX/TX
panels.append({
"id": 12,
"title": "Host Network Receive (RX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_network_receive_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 13,
"title": "Host Network Transmit (TX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (instance) (rate(node_network_transmit_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
# ==========================================
# GROUP 2: Containers & Workloads
# ==========================================
if cluster_key in ["aerodrome", "spaceport"]:
cadvisor_filter = f'cluster="{cluster_key}"' if cluster_key == "spaceport" else 'instance=~"sninfr2.*"'
panels.append({
"collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
"id": 200,
"title": f"📦 Kubernetes Containers & Workloads — {cluster_title} Cluster",
"type": "row"
})
current_y += 1
# Container Stats
panels.extend([
{
"id": 14,
"title": "Monitored Containers",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'count(container_last_seen{{container!="", {cadvisor_filter}}})', "legendFormat": "Containers", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 15,
"title": "Active Kubernetes Pods",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'count(count by (pod, namespace) (container_last_seen{{container!="", {cadvisor_filter}}}))', "legendFormat": "Pods", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 16,
"title": "Total Container CPU Load",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum(rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m]))', "legendFormat": "Cores", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 17,
"title": "Total Container Working RAM",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum(container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "RAM", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
}
])
current_y += 4
# Container CPU
panels.append({
"id": 18,
"title": "Top 10 CPU Consuming Pods (Cores)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 19,
"title": "CPU Throttling by Namespace",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{{{cadvisor_filter}}}[2m]))', "legendFormat": "{{namespace}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
current_y += 8
# Container Memory
panels.append({
"id": 20,
"title": "Top 10 Memory Consuming Pods",
"type": "timeseries",
"gridPos": {"h": 8, "w": 14, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}}))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 21,
"title": "Memory Working Set by Namespace",
"type": "bargauge",
"gridPos": {"h": 8, "w": 10, "x": 14, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'sum by (namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "{{namespace}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}},
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
})
current_y += 8
# Container Network
panels.append({
"id": 22,
"title": "Top Pod Network RX (Receive)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 23,
"title": "Top Pod Network TX (Transmit)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
else:
# Shipyard (Docker Swarm)
panels.append({
"collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
"id": 200,
"title": "📦 Docker Swarm Containers & Service Stacks — Shipyard Fleet",
"type": "row"
})
current_y += 1
panels.extend([
{
"id": 14,
"title": "Monitored Swarm Containers",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'count(container_last_seen{cluster="shipyard", name!=""}) or count(container_last_seen{cluster="shipyard"})', "legendFormat": "Containers", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 15,
"title": "Active Swarm Nodes",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'count(up{cluster="shipyard", job="node-exporter"})', "legendFormat": "Swarm Nodes", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 16,
"title": "Total Container CPU Load",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'sum(rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m]))', "legendFormat": "Cores", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
},
{
"id": 17,
"title": "Total Container Working RAM",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'sum(container_memory_working_set_bytes{cluster="shipyard"})', "legendFormat": "RAM", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
}
])
current_y += 4
# Shipyard CPU & Memory
panels.append({
"id": 18,
"title": "Container CPU Usage by Service/Task",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance, id) (rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
panels.append({
"id": 19,
"title": "Container Memory Working Set by Service/Task",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance, id) (container_memory_working_set_bytes{cluster="shipyard"}))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
})
current_y += 8
# Shipyard Network RX/TX
panels.append({
"id": 20,
"title": "Container Network Receive (RX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
panels.append({
"id": 21,
"title": "Container Network Transmit (TX)",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
"datasource": "Prometheus",
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
})
current_y += 8
dash["panels"] = panels
return dash
clusters = [
{"key": "aerodrome", "title": "Aerodrome", "folder_id": 5, "folder_uid": "folder-aerodrome", "emoji": "✈️"},
{"key": "spaceport", "title": "Spaceport", "folder_id": 6, "folder_uid": "folder-spaceport", "emoji": "🚀"},
{"key": "shipyard", "title": "Shipyard", "folder_id": 7, "folder_uid": "folder-shipyard", "emoji": "⚓"}
]
# 1. Generate and Upload Dashboards
for c in clusters:
dash_json = build_cluster_dashboard(c["key"], c["title"], c["folder_id"], c["folder_uid"], c["emoji"])
# Save to local gitops repo
filename = f"homelab-gitops-repo/dashboards/grafana/{c['key']}-resource-monitor.json"
with open(filename, "w", encoding="utf-8") as f:
json.dump(dash_json, f, indent=2)
print(f"Saved {filename}")
# Upload to Grafana API
payload = {
"dashboard": dash_json,
"folderId": c["folder_id"],
"folderUid": c["folder_uid"],
"overwrite": True
}
resp = requests.post(f"{base_url}/api/dashboards/db", json=payload, auth=auth, verify=False, timeout=10)
print(f"Uploaded {c['title']} dashboard to Grafana: {resp.status_code} - {resp.json().get('status')}")
# 2. Delete old dashboards from AI Stack folder
for old_uid in ["snarfnet-systems", "snarfnet-containers"]:
del_resp = requests.delete(f"{base_url}/api/dashboards/uid/{old_uid}", auth=auth, verify=False, timeout=10)
print(f"Deleted old dashboard {old_uid}: {del_resp.status_code}")
print("\nDashboard generation and deployment complete!")

View File

@@ -0,0 +1,27 @@
import subprocess
import os
edge_path = r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe"
out_png = os.path.abspath("docs/snarfnet_homelab_topology.png")
html_url = "file:///" + os.path.abspath("docs/snarfnet_homelab_topology.html").replace("\\", "/")
cmd = [
edge_path,
"--headless",
"--disable-gpu",
"--no-sandbox",
"--hide-scrollbars",
"--window-size=1920,2400",
f"--screenshot={out_png}",
html_url
]
print("Running cmd:", " ".join(cmd))
res = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
print("Return code:", res.returncode)
print("Stdout:", res.stdout)
print("Stderr:", res.stderr)
print("File exists?", os.path.exists(out_png))
if os.path.exists(out_png):
print("File size:", os.path.getsize(out_png))

10
scripts/cat_21_conf.py Normal file
View File

@@ -0,0 +1,10 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
cmd = "docker exec $(docker ps -q --filter name=nginx-proxy-manager_app) cat /data/nginx/proxy_host/21.conf"
_, o, _ = c.exec_command(cmd)
print(o.read().decode())
c.close()

13
scripts/cat_prom_conf.py Normal file
View File

@@ -0,0 +1,13 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print(run("kubectl exec deployment/prometheus -- cat /etc/prometheus/prometheus.yml"))
c.close()

14
scripts/check_13_conf.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('docker exec $(docker ps -q -f name=nginx-proxy-manager_app) cat /data/nginx/proxy_host/13.conf | grep server_name')
print(stdout.read().decode('utf-8'))
client.close()

25
scripts/check_214_pods.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -o wide")
print("=== PODS IN CATTLE-SYSTEM ===")
print(stdout.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
print("=== ROLLOUT STATUS ===")
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
print("=== ENDPOINT ===")
print(stdout.read().decode('utf-8'))
client.close()

29
scripts/check_215_pods.py Normal file
View File

@@ -0,0 +1,29 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system get pods -o wide")
print("=== PODS IN CATTLE-SYSTEM ===")
print(stdout.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n cattle-system rollout status deployment/rancher --timeout=30s")
print("=== ROLLOUT STATUS ===")
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("curl -k -I -s https://aerodrome.snarfnet.net | head -n 10")
print("=== ENDPOINT ===")
print(stdout.read().decode('utf-8'))
stdin, stdout, stderr = client.exec_command("KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -n cattle-system")
print("=== HELM LIST ===")
print(stdout.read().decode('utf-8'))
client.close()

15
scripts/check_30.py Normal file
View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
c.connect('140.44.4.30', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
print("Connected to 140.44.4.30 with snarf")
stdin, stdout, _ = c.exec_command("hostname && ip a")
print(stdout.read().decode())
c.close()
except Exception as e:
print("Failed 140.44.4.30:", e)

12
scripts/check_33_conf.py Normal file
View File

@@ -0,0 +1,12 @@
import paramiko
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
cmd = 'cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/33.conf'
stdin, stdout, stderr = client.exec_command(cmd)
print("33.conf:\n", stdout.read().decode())
client.close()

View File

@@ -0,0 +1,13 @@
import paramiko
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) tail -n 30 /data/logs/proxy-host-43_access.log")
print("=== Access log 43 (pihole2) around 11:00 ===")
print(stdout.read().decode())
ssh.close()

12
scripts/check_49_conf.py Normal file
View File

@@ -0,0 +1,12 @@
import paramiko
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet", timeout=10)
cmd = 'cat /var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/49.conf'
stdin, stdout, stderr = client.exec_command(cmd)
print("49.conf (ai.snarfnet.net):\n", stdout.read().decode())
client.close()

19
scripts/check_51.py Normal file
View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("ss -tulpn | grep ':80 '")
run("ip a")
run("kubectl get pod -A -o wide | grep sninfr4lnxwrk01 | grep pihole || true")
ssh.close()

18
scripts/check_51_nat.py Normal file
View File

@@ -0,0 +1,18 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.51', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("iptables -t nat -S | grep -E '80\b|31905|31937'")
run("crictl ps")
ssh.close()

View File

@@ -0,0 +1,17 @@
import paramiko
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.71', username='snarf', key_filename=key_path, timeout=10)
stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/55.conf")
print("=== 55.conf (pihole1) ===")
print(stdout.read().decode())
stdin, stdout, stderr = ssh.exec_command("sudo docker exec $(sudo docker ps -q -f name=nginx-proxy-manager) cat /data/nginx/proxy_host/43.conf")
print("=== 43.conf (pihole2) ===")
print(stdout.read().decode())
ssh.close()

View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command('k3s kubectl get pods -n cattle-system')
print("=== AERODROME CATTLE-SYSTEM PODS ===")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get deploy,svc,pod")
print(stdout.read().decode('utf-8'))
c.close()

View File

@@ -0,0 +1,24 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Nodes on Aerodrome:")
print(run("kubectl get nodes -o wide"))
print("\nDaemonSets on Aerodrome:")
print(run("kubectl get ds -A"))
print("\nServices matching metrics:")
print(run("kubectl get svc -A | grep -E 'metric|node|cadvisor|exporter'"))
print("\nKubelet metrics on node 140.44.4.11 (port 10250 or 10255):")
print(run("curl -sk https://localhost:10250/metrics/cadvisor | head -n 10"))
c.close()

View File

@@ -0,0 +1,12 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/pihole -- curl -s -X POST "http://localhost/api/auth" -H "Content-Type: application/json" -d \'{"password":"snarfnetadmin"}\'')
print("Aerodrome API auth test:")
print(stdout.read().decode('utf-8', errors='ignore'))
ssh.close()

14
scripts/check_aero_svc.py Normal file
View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
stdin, stdout, stderr = c.exec_command("k3s kubectl -n infra get svc")
print(stdout.read().decode('utf-8'))
c.close()

View File

@@ -0,0 +1,15 @@
import paramiko
s = paramiko.SSHClient()
s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
_, out, _ = s.exec_command(cmd)
print(out.read().decode())
run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness,STATE:.status.state")
s.close()

View File

@@ -0,0 +1,19 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("kubectl get nodes")
run("kubectl get pods -A | grep -v -E 'Running|Completed' || true")
run("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,ROBUSTNESS:.status.robustness | grep -v healthy || true")
ssh.close()

View File

@@ -0,0 +1,8 @@
import paramiko
ssh=paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_,o,_=ssh.exec_command('kubectl get nodes')
print(o.read().decode())
ssh.close()

View File

@@ -0,0 +1,29 @@
import paramiko
import json
def main():
s = paramiko.SSHClient()
s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
s.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet')
_, o, _ = s.exec_command('kubectl get svc -A -o json')
data = json.loads(o.read().decode())
print("=== AERODROME LOADBALANCER SERVICES ===")
for item in data.get('items', []):
if item.get('spec', {}).get('type') == 'LoadBalancer':
name = item['metadata']['name']
ns = item['metadata']['namespace']
lb_ip = item.get('spec', {}).get('loadBalancerIP')
ing = [x.get('ip') for x in item.get('status', {}).get('loadBalancer', {}).get('ingress', [])]
annotations = item['metadata'].get('annotations', {})
print(f"- {ns}/{name}: status_ingress={ing}, spec_loadBalancerIP={lb_ip}, metallb_annotations={list(annotations.keys())}")
_, o, _ = s.exec_command('kubectl get ipaddresspools.metallb.io -A -o yaml 2>/dev/null || kubectl get cm -n metallb-system config -o yaml 2>/dev/null || true')
print("\n=== METALLB CONFIG ===")
print(o.read().decode())
s.close()
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,29 @@
import paramiko
import sys
import time
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
print("=== CHECKING VETH ON SNINFR0LNXCTL03 ===")
client.connect("140.44.4.43", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("modprobe veth; lsmod | grep veth; systemctl is-active k3s-server || systemctl is-active k3s")
print(stdout.read().decode('utf-8'))
client.close()
time.sleep(5)
print("=== CHECKING PODS IN CATTLE-SYSTEM ON SPACEPORT ===")
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get pods -o wide")
print(stdout.read().decode('utf-8'))
print("=== CHECKING POD EVENTS ===")
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system describe pod -l app=cattle-cluster-agent | grep -A 10 Events:")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system get deployment cattle-cluster-agent -o yaml")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent --tail=100")
print("=== SPACEPORT CATTLE-CLUSTER-AGENT LOGS ===")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,15 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=20")
print("=== AGENT LOGS ===")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,14 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("k3s kubectl -n cattle-system logs -l app=cattle-cluster-agent -c cluster-register --tail=25")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -0,0 +1,19 @@
import json, ssl, urllib.request
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
ZABBIX_URL = 'http://140.44.4.60:8888/api_jsonrpc.php'
payload = {'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': 'Admin', 'password': 'zabbix'}, 'id': 1}
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc'})
token = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
payload = {'jsonrpc': '2.0', 'method': 'alert.get', 'params': {'output': 'extend', 'limit': 20, 'sortfield': 'clock', 'sortorder': 'DESC'}, 'id': 2}
req = urllib.request.Request(ZABBIX_URL, data=json.dumps(payload).encode(), headers={'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {token}'})
alerts = json.loads(urllib.request.urlopen(req, context=ctx).read())['result']
for a in alerts:
print(f"ID: {a['alertid']} | Status: {a['status']} | User: {a['userid']} | Error: '{a.get('error')}' | Subject: {a.get('subject')}")

View File

@@ -0,0 +1,24 @@
import paramiko
nodes = [
("wrk01", "140.44.4.51"),
("wrk02", "140.44.4.52"),
("wrk03", "140.44.4.53"),
("wrk04", "140.44.4.54"),
("wrk05", "140.44.4.55"),
("wrk06", "140.44.4.56"),
]
for name, ip in nodes:
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=4)
_, out, _ = ssh.exec_command("find /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/ -path '*/home/coder/.config/coderv2' 2>/dev/null")
res = out.read().decode().strip()
if res:
print(f"[{name} - {ip}] Found:\n{res}")
ssh.close()
except Exception as e:
print(f"[{name}] err: {e}")

View File

@@ -0,0 +1,19 @@
import subprocess
endpoints = [
("Argo CD", "https://argo.snarfnet.net"),
("Rancher Manager", "https://aerodrome.snarfnet.net"),
("AWX Console", "https://awx.snarfnet.net"),
("Authentik SSO", "https://authentik.snarfnet.net"),
("Uptime Kuma", "https://uptime.snarfnet.net/dashboard"),
("Gitea GitOps", "http://140.44.4.51:90"),
("Zabbix Server", "http://140.44.4.51:8888"),
("Portainer Server", "https://140.44.4.71:9443")
]
print("=== ENDPOINT HEALTH CHECK ===")
for name, url in endpoints:
res = subprocess.run(f"curl.exe -k -s -o nul -w '%{{http_code}}' {url}", capture_output=True, text=True, shell=True)
code = res.stdout.strip()
print(f"[{code}] {name} ({url})")

View File

@@ -0,0 +1,25 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
py_query = """
import sqlite3
conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
cur = conn.cursor()
cur.execute('SELECT id, domain_names, forward_host, forward_port, forward_scheme, enabled FROM proxy_host')
for r in cur.fetchall():
print(r)
conn.close()
"""
stdin, stdout, stderr = c.exec_command(f'python3 -c "{py_query}"')
print("All NPM hosts:\n", stdout.read().decode())
c.close()

View File

@@ -0,0 +1,9 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, o, _ = c.exec_command("kubectl get pods -A")
print(o.read().decode().strip())
c.close()

View File

@@ -0,0 +1,30 @@
import urllib.request
import ssl
import json
import sys
sys.stdout.reconfigure(encoding='utf-8')
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
token = "SnarfNetPermanentAutomationToken2026"
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json"
}
def api_call(url):
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, context=ctx, timeout=8) as resp:
return json.loads(resp.read().decode('utf-8'))
apps = api_call("https://authentik.snarfnet.net/api/v3/core/applications/?page_size=100")
for a in apps.get('results', []):
print(f"App: {a['name']:30} | Slug: {a['slug']:15} | Provider: {a.get('provider')}")
providers = api_call("https://authentik.snarfnet.net/api/v3/providers/proxy/?page_size=100")
for p in providers.get('results', []):
print(f"Proxy Provider: {p['name']:30} | PK: {p['pk']:4} | Mode: {p['mode']:15} | Cookie Domain: {p.get('cookie_domain')}")

27
scripts/check_all_tls.py Normal file
View File

@@ -0,0 +1,27 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Check certificates in Aerodrome across all namespaces
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secrets -A --field-selector=type=kubernetes.io/tls
""")
print("=== TLS SECRETS ON AERODROME ===")
print(stdout.read().decode('utf-8'))
client.close()
# Check certificates on Spaceport across all namespaces
client.connect("140.44.4.41", username="root", key_filename=key_path, timeout=10)
stdin, stdout, stderr = client.exec_command("""
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get secrets -A --field-selector=type=kubernetes.io/tls
""")
print("=== TLS SECRETS ON SPACEPORT ===")
print(stdout.read().decode('utf-8'))
client.close()

View File

@@ -1,17 +1,34 @@
import os
import sys
import paramiko
import sys
sys.stdout.reconfigure(encoding="utf-8")
pkey = paramiko.Ed25519Key.from_private_key_file(os.path.expanduser('~/.ssh/id_ed25519_snarfnet'))
sys.stdout.reconfigure(encoding='utf-8')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.75', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("140.44.4.11", username="root", pkey=pkey, timeout=10)
def run(cmd):
stdin, stdout, stderr = c.exec_command(cmd)
return stdout.read().decode().strip(), stderr.read().decode().strip()
cmd = "curl -s http://140.44.4.21:80/main.105b4163.async.js | grep -o -E '\"/api/[a-zA-Z0-9_/]+\"' | sort -u | head -n 30"
stdin, stdout, stderr = client.exec_command(cmd)
print("API ROUTES ON 140.44.4.21:")
print(stdout.read().decode())
client.close()
cid, _ = run("echo 'Katzir476!' | sudo -S docker ps -q -f name=peanut_peanut")
# Check readdir in src/app or src
script = """
const fs = require('fs');
function walk(dir) {
let res = [];
for (let f of fs.readdirSync(dir)) {
let p = dir + '/' + f;
if (fs.statSync(p).isDirectory()) res.push(...walk(p));
else if (f.endsWith('.ts') || f.endsWith('.js') || f.endsWith('.tsx')) res.push(p);
}
return res;
}
console.log(walk('src/app/api'));
"""
run(f"cat << 'EOF' > /tmp/walk.js\n{script}\nEOF")
out, err = run(f"echo 'Katzir476!' | sudo -S docker exec -i {cid} node < /tmp/walk.js")
print(out)
if err: print(err)
c.close()

View File

@@ -0,0 +1,12 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/technitium-dns -- curl -s "http://localhost:5380/api/status"')
print("api/status response:")
print(stdout.read().decode('utf-8', errors='ignore'))
ssh.close()

25
scripts/check_apis.py Normal file
View File

@@ -0,0 +1,25 @@
import paramiko
import sys
sys.stdout.reconfigure(encoding='utf-8')
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
client.connect("140.44.4.11", username="root", key_filename=key_path, timeout=15)
test_cmd = """
cd /tmp/rancher-test
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm template rancher ./rancher --namespace cattle-system --set hostname=aerodrome.snarfnet.net --set ingress.tls.source=secret | grep "apiVersion:" | sort | uniq -c
"""
stdin, stdout, stderr = client.exec_command(test_cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
print("APIs used:")
print(out)
if err:
print("STDERR:", err)
client.close()

12
scripts/check_apiurl.py Normal file
View File

@@ -0,0 +1,12 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/pihole -- curl -s "http://localhost/admin/login" | grep -i "data-apiurl"')
print("data-apiurl:")
print(stdout.read().decode())
ssh.close()

17
scripts/check_app_dll.py Normal file
View File

@@ -0,0 +1,17 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/technitium-dns -- ls -la /app')
print("/app contents:")
print(stdout.read().decode('utf-8', errors='ignore'))
# Search for AddAuthentication in dlls
stdin, stdout, stderr = ssh.exec_command('k3s kubectl exec -n dns deployment/technitium-dns -- strings /app/DnsServerApp.dll /app/DnsServerCore.dll 2>/dev/null | grep -i -C 3 "AddAuthentication"')
print("AddAuthentication strings:")
print(stdout.read().decode('utf-8', errors='ignore'))
ssh.close()

View File

@@ -0,0 +1,14 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
stdin, stdout, stderr = ssh.exec_command("kubectl get applications -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
ssh.close()

22
scripts/check_argo_ing.py Normal file
View File

@@ -0,0 +1,22 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
print(f"=== {cmd} ===")
stdin, stdout, stderr = ssh.exec_command(cmd)
print(stdout.read().decode())
err = stderr.read().decode()
if err:
print("ERR:", err)
run("kubectl get ingress argo-ing -n infrastructure -o yaml")
run("kubectl get svc pihole-web -n infrastructure -o yaml")
run("kubectl get svc -n infrastructure")
run("kubectl get svc -A | grep -E 'LoadBalancer|140.44.4.62'")
run("curl -k -I -H 'Host: argo.snarfnet.net' http://140.44.4.62")
run("curl -k -I -H 'Host: argo.snarfnet.net' https://140.44.4.62")
ssh.close()

View File

@@ -0,0 +1,9 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', password='dietpi', timeout=10)
_, o, e = c.exec_command("kubectl get app aerodrome-prometheus -n infrastructure")
print("out:", o.read().decode().strip())
print("err:", e.read().decode().strip())
c.close()

View File

@@ -0,0 +1,16 @@
import paramiko
import base64
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
def run(cmd):
_, o, e = c.exec_command(cmd)
return o.read().decode().strip()
print("Argo initial secret:")
print(run("kubectl get secret argocd-initial-admin-secret -n infrastructure -o jsonpath='{.data.password}'"))
c.close()

View File

@@ -0,0 +1,8 @@
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('140.44.4.41', username='root', password='dietpi', timeout=10)
_, o, _ = c.exec_command("kubectl get applications.argoproj.io -n infrastructure")
print(o.read().decode().strip())
c.close()

View File

@@ -0,0 +1,11 @@
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('140.44.4.41', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
_, out, _ = ssh.exec_command("kubectl get applications -n infrastructure -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status")
print(out.read().decode())
ssh.close()

Some files were not shown because too many files have changed in this diff Show More