feat: add various infrastructure check and automation scripts
This commit is contained in:
62
scripts/add_kuma_spacedock.py
Normal file
62
scripts/add_kuma_spacedock.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run_sql(pod, sql):
|
||||
cmd = f"kubectl exec -i -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root"
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
stdin.write(sql)
|
||||
stdin.close()
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
return out, err
|
||||
|
||||
_, o, _ = c.exec_command("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
||||
pod = o.read().decode().strip()
|
||||
print("Kuma Pod:", pod)
|
||||
|
||||
monitors = [
|
||||
("Spacedock Harvester Node 01 (140.44.4.81)", "140.44.4.81"),
|
||||
("Spacedock Harvester Node 02 (140.44.4.82)", "140.44.4.82"),
|
||||
("Spacedock Harvester Node 03 (140.44.4.83)", "140.44.4.83"),
|
||||
("Spacedock Harvester Node 04 (140.44.4.84)", "140.44.4.84"),
|
||||
("Dell PowerEdge R730 Node 1 iDRAC (140.44.44.121)", "140.44.44.121"),
|
||||
("Dell PowerEdge R730 Node 2 iDRAC (140.44.44.122)", "140.44.44.122"),
|
||||
("Dell PowerEdge R730 Node 3 iDRAC (140.44.44.123)", "140.44.44.123"),
|
||||
("Dell PowerEdge R730 Node 4 iDRAC (140.44.44.124)", "140.44.44.124")
|
||||
]
|
||||
|
||||
for name, host in monitors:
|
||||
out, err = run_sql(pod, f"USE kuma; SELECT id FROM monitor WHERE hostname='{host}' AND type='ping';")
|
||||
if "id" in out and len(out.splitlines()) > 1:
|
||||
mon_id = out.splitlines()[1].strip()
|
||||
print(f"Monitor already exists: {name} (ID: {mon_id})")
|
||||
else:
|
||||
insert_sql = f"""USE kuma;
|
||||
INSERT INTO monitor (name, active, user_id, `interval`, type, weight, hostname, maxretries, retry_interval)
|
||||
VALUES ('{name}', 1, 1, 60, 'ping', 2000, '{host}', 1, 60);
|
||||
SELECT LAST_INSERT_ID() AS id;
|
||||
"""
|
||||
out, err = run_sql(pod, insert_sql)
|
||||
if err:
|
||||
print(f"Error inserting {name}: {err}")
|
||||
continue
|
||||
lines = out.splitlines()
|
||||
mon_id = lines[-1].strip()
|
||||
print(f"Created monitor: {name} (ID: {mon_id})")
|
||||
|
||||
# Link notifications (1 = Discord, 2 = Gotify)
|
||||
for notif_id in [1, 2]:
|
||||
link_check = f"USE kuma; SELECT id FROM monitor_notification WHERE monitor_id={mon_id} AND notification_id={notif_id};"
|
||||
out_link, _ = run_sql(pod, link_check)
|
||||
if "id" not in out_link or len(out_link.splitlines()) <= 1:
|
||||
run_sql(pod, f"USE kuma; INSERT INTO monitor_notification (monitor_id, notification_id) VALUES ({mon_id}, {notif_id});")
|
||||
print(f" Linked monitor {mon_id} to notification {notif_id}")
|
||||
else:
|
||||
print(f" Monitor {mon_id} already linked to notification {notif_id}")
|
||||
|
||||
c.close()
|
||||
173
scripts/add_npm_games.py
Normal file
173
scripts/add_npm_games.py
Normal file
@@ -0,0 +1,173 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import datetime
|
||||
import json
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=10)
|
||||
|
||||
def main():
|
||||
remote_script = """import sqlite3, datetime, json
|
||||
|
||||
db_path = '/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Check existing games host
|
||||
cur.execute("SELECT id, domain_names, forward_host, forward_port FROM proxy_host WHERE domain_names LIKE '%games.snarfnet.net%'")
|
||||
existing = cur.fetchone()
|
||||
|
||||
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
if existing:
|
||||
pid = existing[0]
|
||||
print(f"Updating existing proxy_host ID {pid}...")
|
||||
cur.execute('''UPDATE proxy_host
|
||||
SET forward_host = '140.44.44.80', forward_port = 8080, forward_scheme = 'http',
|
||||
certificate_id = 1, ssl_forced = 1, hsts_enabled = 1, http2_support = 1,
|
||||
block_exploits = 1, allow_websocket_upgrade = 1, enabled = 1, modified_on = ?
|
||||
WHERE id = ?''', (now, pid))
|
||||
else:
|
||||
cur.execute("SELECT MAX(id) FROM proxy_host")
|
||||
max_id = cur.fetchone()[0] or 0
|
||||
pid = max_id + 1
|
||||
print(f"Creating new proxy_host with ID {pid}...")
|
||||
cur.execute('''INSERT INTO proxy_host (
|
||||
id, created_on, modified_on, owner_user_id, domain_names,
|
||||
forward_host, forward_port, forward_scheme, certificate_id,
|
||||
ssl_forced, hsts_enabled, hsts_subdomains, http2_support,
|
||||
block_exploits, caching_enabled, allow_websocket_upgrade,
|
||||
access_list_id, advanced_config, enabled, meta
|
||||
) VALUES (
|
||||
?, ?, ?, 1, '["games.snarfnet.net"]',
|
||||
'140.44.44.80', 8080, 'http', 1,
|
||||
1, 1, 0, 1,
|
||||
1, 0, 1,
|
||||
0, '', 1, '{}'
|
||||
)''', (pid, now, now))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"SUCCESS: pid={pid}")
|
||||
"""
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open("/tmp/add_games_proxy.py", "w") as f:
|
||||
f.write(remote_script)
|
||||
sftp.close()
|
||||
|
||||
stdin, stdout, stderr = c.exec_command("python3 /tmp/add_games_proxy.py")
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
print("DB Result:\n", out)
|
||||
if err:
|
||||
print("DB Err:\n", err)
|
||||
|
||||
# Extract pid
|
||||
pid = None
|
||||
for line in out.splitlines():
|
||||
if "pid=" in line:
|
||||
pid = line.split("pid=")[1].strip()
|
||||
|
||||
if not pid:
|
||||
print("Failed to get pid!")
|
||||
return
|
||||
|
||||
# Write nginx config
|
||||
nginx_conf = f"""# ------------------------------------------------------------
|
||||
# games.snarfnet.net
|
||||
# ------------------------------------------------------------
|
||||
|
||||
map $scheme $hsts_header {{
|
||||
https "max-age=63072000; preload";
|
||||
}}
|
||||
|
||||
server {{
|
||||
set $forward_scheme http;
|
||||
set $server "140.44.44.80";
|
||||
set $port 8080;
|
||||
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
|
||||
server_name games.snarfnet.net;
|
||||
http2 on;
|
||||
|
||||
# Let's Encrypt SSL (*.snarfnet.net is cert 1)
|
||||
include conf.d/include/letsencrypt-acme-challenge.conf;
|
||||
include conf.d/include/ssl-cache.conf;
|
||||
include conf.d/include/ssl-ciphers.conf;
|
||||
ssl_certificate /etc/letsencrypt/live/npm-1/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/npm-1/privkey.pem;
|
||||
|
||||
# Asset Caching
|
||||
include conf.d/include/assets.conf;
|
||||
|
||||
# Block Exploits
|
||||
include conf.d/include/block-exploits.conf;
|
||||
|
||||
# Force SSL
|
||||
set $trust_forwarded_proto "F";
|
||||
include conf.d/include/force-ssl.conf;
|
||||
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $http_connection;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
access_log /data/logs/proxy-host-{pid}_access.log proxy;
|
||||
error_log /data/logs/proxy-host-{pid}_error.log warn;
|
||||
|
||||
location / {{
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $http_connection;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
# Proxy!
|
||||
include conf.d/include/proxy.conf;
|
||||
}}
|
||||
|
||||
# Custom
|
||||
include /data/nginx/custom/server_proxy[.]conf;
|
||||
}}
|
||||
"""
|
||||
conf_path = f"/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/nginx/proxy_host/{pid}.conf"
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open(conf_path, "w") as f:
|
||||
f.write(nginx_conf)
|
||||
sftp.close()
|
||||
print(f"Wrote {conf_path}")
|
||||
|
||||
# Test Nginx
|
||||
stdin, stdout, stderr = c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -t")
|
||||
t_out = stdout.read().decode()
|
||||
t_err = stderr.read().decode()
|
||||
print("nginx -t stdout:\n", t_out)
|
||||
print("nginx -t stderr:\n", t_err)
|
||||
|
||||
if "successful" in t_out or "successful" in t_err:
|
||||
print("Reloading Nginx in container...")
|
||||
c.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -s reload")
|
||||
# Also check other swarm node if running dual
|
||||
c2 = paramiko.SSHClient()
|
||||
c2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
c2.connect("140.44.4.72", username="root", key_filename=key_path, timeout=5)
|
||||
c2.exec_command("docker exec $(docker ps -q -f name=nginx-proxy-manager_app | head -n1) nginx -s reload")
|
||||
c2.close()
|
||||
print("Reloaded Swarm 02 NPM as well!")
|
||||
except Exception as e:
|
||||
print("Swarm 02 reload skipped/note:", e)
|
||||
print("Proxy setup complete!")
|
||||
else:
|
||||
print("ERROR: nginx -t failed!")
|
||||
|
||||
c.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
67
scripts/adjust_zabbix_thresholds.py
Normal file
67
scripts/adjust_zabbix_thresholds.py
Normal file
@@ -0,0 +1,67 @@
|
||||
import urllib.request
|
||||
import json
|
||||
import ssl
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def zabbix_api(method, params, auth=None):
|
||||
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
|
||||
if auth: payload["auth"] = auth
|
||||
req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php",
|
||||
data=json.dumps(payload).encode('utf-8'),
|
||||
headers={"Content-Type": "application/json-rpc"})
|
||||
resp = urllib.request.urlopen(req, context=ctx, timeout=10)
|
||||
return json.loads(resp.read().decode('utf-8'))
|
||||
|
||||
auth = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"}).get("result")
|
||||
print("Zabbix Auth OK:", bool(auth))
|
||||
|
||||
# 1. Update Template Macros for 10001 (Agent passive), 10343 (Agent active), 10285 (Prom)
|
||||
# Find hostmacroids for these templates
|
||||
template_ids = ["10001", "10343", "10285"]
|
||||
for tid in template_ids:
|
||||
macros = zabbix_api("usermacro.get", {
|
||||
"output": ["hostmacroid", "macro", "value"],
|
||||
"hostids": [tid]
|
||||
}, auth=auth).get("result", [])
|
||||
for m in macros:
|
||||
mid = m["hostmacroid"]
|
||||
macro_name = m["macro"]
|
||||
if macro_name == "{$CPU.UTIL.CRIT}":
|
||||
res = zabbix_api("usermacro.update", {"hostmacroid": mid, "value": "95"}, auth=auth)
|
||||
print(f"Template {tid} macro {macro_name} updated to 95: {res.get('result')}")
|
||||
elif macro_name == "{$MEMORY.UTIL.MAX}":
|
||||
res = zabbix_api("usermacro.update", {"hostmacroid": mid, "value": "95"}, auth=auth)
|
||||
print(f"Template {tid} macro {macro_name} updated to 95: {res.get('result')}")
|
||||
elif macro_name == "{$LOAD_AVG_PER_CPU.MAX.WARN}":
|
||||
res = zabbix_api("usermacro.update", {"hostmacroid": mid, "value": "2.5"}, auth=auth)
|
||||
print(f"Template {tid} macro {macro_name} updated to 2.5: {res.get('result')}")
|
||||
|
||||
# 2. Update trigger expressions from 5m to 15m
|
||||
# Triggers: 22370 (Passive CPU), 22376 (Passive Mem), 22415 (Active CPU), 22421 (Active Mem)
|
||||
# In Zabbix 6.0/7.0, expression format is min(/<template_name>/<item_key>,15m)>{$MACRO}
|
||||
# Let's inspect the exact expression syntax for trigger update
|
||||
for tr_id in ["22370", "22376", "22415", "22421"]:
|
||||
tr = zabbix_api("trigger.get", {
|
||||
"output": ["triggerid", "description", "expression"],
|
||||
"triggerids": [tr_id]
|
||||
}, auth=auth).get("result", [{}])[0]
|
||||
expr = tr.get("expression", "")
|
||||
print(f"\nTrigger {tr_id} ({tr.get('description')}): current expr = {expr}")
|
||||
# Replace 5m with 15m in the expression
|
||||
if "5m" in expr:
|
||||
new_expr = expr.replace("5m", "15m")
|
||||
# In trigger.update, Zabbix expects expression with host/item syntax if updated
|
||||
# Or description update: "Linux: High CPU utilization (>95% for 15m)"
|
||||
desc = tr.get("description", "")
|
||||
# Update trigger
|
||||
up_res = zabbix_api("trigger.update", {
|
||||
"triggerid": tr_id,
|
||||
"expression": new_expr
|
||||
}, auth=auth)
|
||||
print(f" Updated expr to 15m: {up_res}")
|
||||
|
||||
print("\nFinished updating Zabbix noise reduction thresholds!")
|
||||
|
||||
37
scripts/apply_clean_cm.py
Normal file
37
scripts/apply_clean_cm.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
|
||||
# Extract just the ConfigMap section
|
||||
# It starts with "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: prometheus-config" and ends with "---"
|
||||
parts = text.split("---")
|
||||
cm_part = None
|
||||
for p in parts:
|
||||
if "name: prometheus-config" in p:
|
||||
cm_part = p
|
||||
break
|
||||
|
||||
if cm_part:
|
||||
print("Found prometheus-config ConfigMap! Size:", len(cm_part))
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/prom_cm_clean.yaml")
|
||||
stdin.write(cm_part)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
_, o, e = c.exec_command("kubectl apply -f /tmp/prom_cm_clean.yaml")
|
||||
print("Apply result:\n", o.read().decode().strip())
|
||||
err = e.read().decode().strip()
|
||||
if err:
|
||||
print("Apply err:\n", err)
|
||||
|
||||
# Reload prometheus via lifecycle endpoint
|
||||
_, o, _ = c.exec_command("kubectl exec deploy/prometheus -- wget -qO- --post-action='' http://localhost:9090/-/reload")
|
||||
print("Reload endpoint output:", o.read().decode().strip())
|
||||
|
||||
c.close()
|
||||
|
||||
79
scripts/apply_kuma_tags.py
Normal file
79
scripts/apply_kuma_tags.py
Normal file
@@ -0,0 +1,79 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run_sql(sql):
|
||||
pod_cmd = "kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'"
|
||||
_, o, _ = c.exec_command(pod_cmd)
|
||||
pod = o.read().decode().strip()
|
||||
cmd = f"kubectl exec -i -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root"
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
stdin.write(sql)
|
||||
stdin.close()
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
# Target monitors:
|
||||
# 98: Spacedock Harvester Node 01
|
||||
# 99: Spacedock Harvester Node 02
|
||||
# 100: Spacedock Harvester Node 03
|
||||
# 101: Spacedock Harvester Node 04
|
||||
# 102: Dell PowerEdge R730 Node 1 iDRAC
|
||||
# 103: Dell PowerEdge R730 Node 2 iDRAC
|
||||
# 104: Dell PowerEdge R730 Node 3 iDRAC
|
||||
# 105: Dell PowerEdge R730 Node 4 iDRAC
|
||||
|
||||
# Tags to attach:
|
||||
# 5: ai
|
||||
# 9: IP
|
||||
# 11: Tier-2-Operational
|
||||
|
||||
monitors = [98, 99, 100, 101, 102, 103, 104, 105]
|
||||
tag_ids = [5, 9, 11]
|
||||
|
||||
sql_commands = ["USE kuma;"]
|
||||
|
||||
for m_id in monitors:
|
||||
for t_id in tag_ids:
|
||||
# Check if already exists
|
||||
check = f"SELECT id FROM monitor_tag WHERE monitor_id={m_id} AND tag_id={t_id};"
|
||||
sql_commands.append(f"""
|
||||
INSERT INTO monitor_tag (monitor_id, tag_id, value)
|
||||
SELECT {m_id}, {t_id}, NULL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM monitor_tag WHERE monitor_id={m_id} AND tag_id={t_id});
|
||||
""")
|
||||
|
||||
# Also ensure Harvester Console (57) and Harvester HCI Console VIP (94) have ai (5)
|
||||
for m_id in [57, 94]:
|
||||
sql_commands.append(f"""
|
||||
INSERT INTO monitor_tag (monitor_id, tag_id, value)
|
||||
SELECT {m_id}, 5, NULL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM monitor_tag WHERE monitor_id={m_id} AND tag_id=5);
|
||||
""")
|
||||
|
||||
full_sql = "\n".join(sql_commands)
|
||||
out, err = run_sql(full_sql)
|
||||
if err:
|
||||
print("SQL Error:", err)
|
||||
else:
|
||||
print("Tags applied successfully!")
|
||||
|
||||
# Verification query
|
||||
verify_sql = """
|
||||
USE kuma;
|
||||
SELECT m.id, m.name, GROUP_CONCAT(t.name ORDER BY t.id SEPARATOR ', ') AS tags
|
||||
FROM monitor m
|
||||
JOIN monitor_tag mt ON m.id = mt.monitor_id
|
||||
JOIN tag t ON mt.tag_id = t.id
|
||||
WHERE m.id IN (57, 94, 98, 99, 100, 101, 102, 103, 104, 105)
|
||||
GROUP BY m.id, m.name
|
||||
ORDER BY m.id;
|
||||
"""
|
||||
res, _ = run_sql(verify_sql)
|
||||
print("\nUpdated Monitors and Tags:\n", res)
|
||||
|
||||
c.close()
|
||||
|
||||
38
scripts/apply_prom_cm_only.py
Normal file
38
scripts/apply_prom_cm_only.py
Normal file
@@ -0,0 +1,38 @@
|
||||
import paramiko
|
||||
import yaml
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r') as f:
|
||||
docs = list(yaml.safe_load_all(f))
|
||||
|
||||
# Find ConfigMap prometheus-config
|
||||
cm_doc = None
|
||||
for d in docs:
|
||||
if d and d.get('kind') == 'ConfigMap' and d.get('metadata', {}).get('name') == 'prometheus-config':
|
||||
cm_doc = d
|
||||
break
|
||||
|
||||
if cm_doc:
|
||||
cm_yaml = yaml.dump(cm_doc)
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus_cm.yaml")
|
||||
stdin.write(cm_yaml)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
_, o, e = c.exec_command("kubectl apply -f /tmp/prometheus_cm.yaml")
|
||||
print("CM apply:\n", o.read().decode().strip())
|
||||
err = e.read().decode().strip()
|
||||
if err:
|
||||
print("CM err:", err)
|
||||
|
||||
# Restart prometheus pod
|
||||
_, o, _ = c.exec_command("kubectl rollout restart deployment/prometheus")
|
||||
print("Prometheus restart:", o.read().decode().strip())
|
||||
_, o, _ = c.exec_command("kubectl rollout status deployment/prometheus --timeout=60s")
|
||||
print("Rollout status:", o.read().decode().strip())
|
||||
|
||||
c.close()
|
||||
|
||||
36
scripts/apply_prom_shipyard.py
Normal file
36
scripts/apply_prom_shipyard.py
Normal file
@@ -0,0 +1,36 @@
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r') as f:
|
||||
prom_yaml = f.read()
|
||||
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/prometheus.yaml")
|
||||
stdin.write(prom_yaml)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
_, o, e = c.exec_command("kubectl apply -f /tmp/prometheus.yaml")
|
||||
print("Kubectl apply:\n", o.read().decode().strip())
|
||||
err = e.read().decode().strip()
|
||||
if err:
|
||||
print("Err:", err)
|
||||
|
||||
# Find prometheus pod
|
||||
_, o, _ = c.exec_command("kubectl get pod -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}'")
|
||||
pod = o.read().decode().strip()
|
||||
print("Prometheus pod:", pod)
|
||||
|
||||
# Reload config
|
||||
_, o, _ = c.exec_command(f"kubectl exec {pod} -- wget -qO- --post-action='' http://localhost:9090/-/reload || kubectl exec {pod} -- curl -X POST http://localhost:9090/-/reload")
|
||||
print("Reload result:", o.read().decode().strip())
|
||||
|
||||
time.sleep(5)
|
||||
# Check targets for shipyard
|
||||
_, o, _ = c.exec_command(f"""kubectl exec {pod} -- wget -qO- 'http://localhost:9090/api/v1/targets' | grep -o '"cluster":"shipyard"[^}}]*' | head -10""")
|
||||
print("Shipyard targets found in Prometheus:\n", o.read().decode().strip())
|
||||
|
||||
c.close()
|
||||
84
scripts/audit_spacedock_failure.py
Normal file
84
scripts/audit_spacedock_failure.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
HARVESTER_VIP = "140.44.4.90"
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
try:
|
||||
print(f"Connecting to Harvester VIP {HARVESTER_VIP}...")
|
||||
c.connect(HARVESTER_VIP, username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
print("Connected to Harvester VIP!")
|
||||
|
||||
def run_k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
stdin, stdout, stderr = c.exec_command(full_cmd)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
return out
|
||||
|
||||
print("\n=== 1. HARVESTER CLUSTER NODES ===")
|
||||
print(run_k8s("kubectl get nodes -o wide"))
|
||||
|
||||
print("\n=== 2. NODE UPTIME ===")
|
||||
print(run_k8s("uptime"))
|
||||
|
||||
print("\n=== 3. VIRTUAL MACHINES (KubeVirt) ===")
|
||||
print(run_k8s("kubectl get vms,vmis -A -o wide"))
|
||||
|
||||
print("\n=== 4. HARVESTER SYSTEM POD STATUS (Not Running / Degraded) ===")
|
||||
print(run_k8s("kubectl get pods -A | grep -v 'Running\\|Completed'"))
|
||||
|
||||
print("\n=== 5. LONGHORN STORAGE VOLUME STATUS ===")
|
||||
print(run_k8s("kubectl get volumes.longhorn.io -n longhorn-system"))
|
||||
|
||||
c.close()
|
||||
except Exception as e:
|
||||
print(f"Error connecting to Harvester: {e}")
|
||||
|
||||
# Check iDRACs
|
||||
idracs = {
|
||||
"Spacedock Node 1": "140.44.44.121",
|
||||
"Spacedock Node 2": "140.44.44.122",
|
||||
"Spacedock Node 3": "140.44.44.123",
|
||||
"Spacedock Node 4": "140.44.44.124",
|
||||
}
|
||||
|
||||
print("\n=== 6. IDRAC SYSTEM EVENT LOGS & POWER STATUS ===")
|
||||
for name, ip in idracs.items():
|
||||
print(f"\n--- {name} ({ip}) ---")
|
||||
try:
|
||||
ic = paramiko.SSHClient()
|
||||
ic.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ic.connect(ip, username="root", password="Katzir476!", timeout=8)
|
||||
chan = ic.invoke_shell()
|
||||
time.sleep(1.5)
|
||||
if chan.recv_ready():
|
||||
chan.recv(4096)
|
||||
|
||||
def racadm(cmd):
|
||||
chan.send(cmd + "\r\n")
|
||||
time.sleep(2)
|
||||
out = ""
|
||||
while chan.recv_ready():
|
||||
out += chan.recv(8192).decode('utf-8', errors='ignore')
|
||||
time.sleep(0.5)
|
||||
return out
|
||||
|
||||
pwr = racadm("racadm serveraction powerstatus")
|
||||
for line in pwr.splitlines():
|
||||
if "power" in line.lower() or "server" in line.lower():
|
||||
print(line.strip())
|
||||
|
||||
sel = racadm("racadm getsel -s 10")
|
||||
print("Last 10 SEL Events:")
|
||||
for line in sel.splitlines()[-12:]:
|
||||
print(" ", line.strip())
|
||||
|
||||
ic.close()
|
||||
except Exception as e:
|
||||
print(f"Error querying {name}: {e}")
|
||||
|
||||
467
scripts/build_and_upload_dashboards.py
Normal file
467
scripts/build_and_upload_dashboards.py
Normal file
@@ -0,0 +1,467 @@
|
||||
import json
|
||||
import requests
|
||||
import copy
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
base_url = "https://grafana.snarfnet.net"
|
||||
|
||||
# Load templates
|
||||
with open("homelab-gitops-repo/dashboards/grafana/systems-dashboard.json") as f:
|
||||
sys_template = json.load(f)
|
||||
|
||||
with open("homelab-gitops-repo/dashboards/grafana/container-metrics-dashboard.json") as f:
|
||||
cnt_template = json.load(f)
|
||||
|
||||
def build_cluster_dashboard(cluster_key, cluster_title, folder_id, folder_uid, cluster_emoji):
|
||||
dash = {
|
||||
"annotations": {"list": []},
|
||||
"editable": True,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": None,
|
||||
"links": [],
|
||||
"liveNow": False,
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [cluster_key, "resource-monitor", "infrastructure"],
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"timepicker": {"refresh_intervals": ["5s", "10s", "15s", "30s", "1m", "5m"]},
|
||||
"timezone": "browser",
|
||||
"title": f"{cluster_emoji} {cluster_title} Resource Monitor",
|
||||
"uid": f"{cluster_key}-resource-monitor",
|
||||
"version": 1,
|
||||
"panels": []
|
||||
}
|
||||
|
||||
panels = []
|
||||
current_y = 0
|
||||
|
||||
# ==========================================
|
||||
# GROUP 1: Systems & Cluster Host Performance
|
||||
# ==========================================
|
||||
# Elevate Systems Row Header
|
||||
panels.append({
|
||||
"collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
|
||||
"id": 100,
|
||||
"title": f"🖥️ Systems & Host Performance — {cluster_title} Cluster Fleet",
|
||||
"type": "row"
|
||||
})
|
||||
current_y += 1
|
||||
|
||||
# Systems Stat Panels (height 4)
|
||||
stat_panels = [
|
||||
{
|
||||
"id": 1,
|
||||
"title": f"Online Nodes ({cluster_title})",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'count(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Nodes", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Avg Cluster CPU Load (1m)",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'avg(node_load1{{cluster="{cluster_key}"}})', "legendFormat": "Load", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}, {"color": "orange", "value": 2.0}, {"color": "red", "value": 4.0}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Total Monitored RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})', "legendFormat": "RAM", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Cluster RAM In-Use %",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'(1 - (sum(node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}) / sum(node_memory_MemTotal_bytes{{cluster="{cluster_key}"}}))) * 100', "legendFormat": "In-Use", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
}
|
||||
]
|
||||
panels.extend(stat_panels)
|
||||
current_y += 4
|
||||
|
||||
# Systems CPU Timeseries
|
||||
panels.append({
|
||||
"id": 5,
|
||||
"title": "CPU Utilization by Host (%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'100 - (avg by (instance) (rate(node_cpu_seconds_total{{mode="idle", cluster="{cluster_key}"}}[2m])) * 100)', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 6,
|
||||
"title": "1-Minute Load Average by Host",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'node_load1{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none", "min": 0}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Systems Memory Timeseries
|
||||
panels.append({
|
||||
"id": 7,
|
||||
"title": "Memory Used % by Host",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'(1 - (node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}} / node_memory_MemTotal_bytes{{cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "percent", "min": 0, "max": 100}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 8,
|
||||
"title": "Available RAM by Host (GiB)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'node_memory_MemAvailable_bytes{{cluster="{cluster_key}"}}', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "bytes"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "min"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Filesystem & Disk I/O
|
||||
panels.append({
|
||||
"id": 9,
|
||||
"title": "Root (/) Filesystem Usage %",
|
||||
"type": "bargauge",
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'(1 - (node_filesystem_avail_bytes{{mountpoint="/", cluster="{cluster_key}"}} / node_filesystem_size_bytes{{mountpoint="/", cluster="{cluster_key}"}})) * 100', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, {"color": "orange", "value": 75}, {"color": "red", "value": 90}]}, "unit": "percent", "min": 0, "max": 100}},
|
||||
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
|
||||
})
|
||||
panels.append({
|
||||
"id": 10,
|
||||
"title": "Disk Read Throughput",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_disk_read_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 11,
|
||||
"title": "Disk Write Throughput",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_disk_written_bytes_total{{cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Network RX/TX
|
||||
panels.append({
|
||||
"id": 12,
|
||||
"title": "Host Network Receive (RX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_network_receive_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 13,
|
||||
"title": "Host Network Transmit (TX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (instance) (rate(node_network_transmit_bytes_total{{device!~"lo|veth.*|flannel.*|cali.*", cluster="{cluster_key}"}}[2m]))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# ==========================================
|
||||
# GROUP 2: Containers & Workloads
|
||||
# ==========================================
|
||||
if cluster_key in ["aerodrome", "spaceport"]:
|
||||
cadvisor_filter = f'cluster="{cluster_key}"' if cluster_key == "spaceport" else 'instance=~"sninfr2.*"'
|
||||
|
||||
panels.append({
|
||||
"collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
|
||||
"id": 200,
|
||||
"title": f"📦 Kubernetes Containers & Workloads — {cluster_title} Cluster",
|
||||
"type": "row"
|
||||
})
|
||||
current_y += 1
|
||||
|
||||
# Container Stats
|
||||
panels.extend([
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Monitored Containers",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'count(container_last_seen{{container!="", {cadvisor_filter}}})', "legendFormat": "Containers", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Active Kubernetes Pods",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'count(count by (pod, namespace) (container_last_seen{{container!="", {cadvisor_filter}}}))', "legendFormat": "Pods", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Total Container CPU Load",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum(rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m]))', "legendFormat": "Cores", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Total Container Working RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum(container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "RAM", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
}
|
||||
])
|
||||
current_y += 4
|
||||
|
||||
# Container CPU
|
||||
panels.append({
|
||||
"id": 18,
|
||||
"title": "Top 10 CPU Consuming Pods (Cores)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{{container!="", {cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 19,
|
||||
"title": "CPU Throttling by Namespace",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{{{cadvisor_filter}}}[2m]))', "legendFormat": "{{namespace}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10}, "unit": "none"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Container Memory
|
||||
panels.append({
|
||||
"id": 20,
|
||||
"title": "Top 10 Memory Consuming Pods",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 14, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}}))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 21,
|
||||
"title": "Memory Working Set by Namespace",
|
||||
"type": "bargauge",
|
||||
"gridPos": {"h": 8, "w": 10, "x": 14, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'sum by (namespace) (container_memory_working_set_bytes{{container!="", {cadvisor_filter}}})', "legendFormat": "{{namespace}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}},
|
||||
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": True}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Container Network
|
||||
panels.append({
|
||||
"id": 22,
|
||||
"title": "Top Pod Network RX (Receive)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_receive_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 23,
|
||||
"title": "Top Pod Network TX (Transmit)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": f'topk(10, sum by (pod, namespace) (rate(container_network_transmit_bytes_total{{{cadvisor_filter}}}[2m])))', "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
else:
|
||||
# Shipyard (Docker Swarm)
|
||||
panels.append({
|
||||
"collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": current_y},
|
||||
"id": 200,
|
||||
"title": "📦 Docker Swarm Containers & Service Stacks — Shipyard Fleet",
|
||||
"type": "row"
|
||||
})
|
||||
current_y += 1
|
||||
|
||||
panels.extend([
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Monitored Swarm Containers",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'count(container_last_seen{cluster="shipyard", name!=""}) or count(container_last_seen{cluster="shipyard"})', "legendFormat": "Containers", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Active Swarm Nodes",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'count(up{cluster="shipyard", job="node-exporter"})', "legendFormat": "Swarm Nodes", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Total Container CPU Load",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'sum(rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m]))', "legendFormat": "Cores", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "orange", "value": None}]}, "unit": "none"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Total Container Working RAM",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'sum(container_memory_working_set_bytes{cluster="shipyard"})', "legendFormat": "RAM", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": None}]}, "unit": "bytes"}},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "center"}
|
||||
}
|
||||
])
|
||||
current_y += 4
|
||||
|
||||
# Shipyard CPU & Memory
|
||||
panels.append({
|
||||
"id": 18,
|
||||
"title": "Container CPU Usage by Service/Task",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance, id) (rate(container_cpu_usage_seconds_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "none"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 19,
|
||||
"title": "Container Memory Working Set by Service/Task",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance, id) (container_memory_working_set_bytes{cluster="shipyard"}))', "legendFormat": "{{instance}} ({{id}})", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "bytes"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
# Shipyard Network RX/TX
|
||||
panels.append({
|
||||
"id": 20,
|
||||
"title": "Container Network Receive (RX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_receive_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
panels.append({
|
||||
"id": 21,
|
||||
"title": "Container Network Transmit (TX)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": current_y},
|
||||
"datasource": "Prometheus",
|
||||
"targets": [{"expr": 'topk(10, sum by (instance) (rate(container_network_transmit_bytes_total{cluster="shipyard"}[2m])))', "legendFormat": "{{instance}}", "refId": "A"}],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15}, "unit": "Bps"}},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}}
|
||||
})
|
||||
current_y += 8
|
||||
|
||||
dash["panels"] = panels
|
||||
return dash
|
||||
|
||||
clusters = [
|
||||
{"key": "aerodrome", "title": "Aerodrome", "folder_id": 5, "folder_uid": "folder-aerodrome", "emoji": "✈️"},
|
||||
{"key": "spaceport", "title": "Spaceport", "folder_id": 6, "folder_uid": "folder-spaceport", "emoji": "🚀"},
|
||||
{"key": "shipyard", "title": "Shipyard", "folder_id": 7, "folder_uid": "folder-shipyard", "emoji": "⚓"}
|
||||
]
|
||||
|
||||
# 1. Generate and Upload Dashboards
|
||||
for c in clusters:
|
||||
dash_json = build_cluster_dashboard(c["key"], c["title"], c["folder_id"], c["folder_uid"], c["emoji"])
|
||||
|
||||
# Save to local gitops repo
|
||||
filename = f"homelab-gitops-repo/dashboards/grafana/{c['key']}-resource-monitor.json"
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
json.dump(dash_json, f, indent=2)
|
||||
print(f"Saved {filename}")
|
||||
|
||||
# Upload to Grafana API
|
||||
payload = {
|
||||
"dashboard": dash_json,
|
||||
"folderId": c["folder_id"],
|
||||
"folderUid": c["folder_uid"],
|
||||
"overwrite": True
|
||||
}
|
||||
resp = requests.post(f"{base_url}/api/dashboards/db", json=payload, auth=auth, verify=False, timeout=10)
|
||||
print(f"Uploaded {c['title']} dashboard to Grafana: {resp.status_code} - {resp.json().get('status')}")
|
||||
|
||||
# 2. Delete old dashboards from AI Stack folder
|
||||
for old_uid in ["snarfnet-systems", "snarfnet-containers"]:
|
||||
del_resp = requests.delete(f"{base_url}/api/dashboards/uid/{old_uid}", auth=auth, verify=False, timeout=10)
|
||||
print(f"Deleted old dashboard {old_uid}: {del_resp.status_code}")
|
||||
|
||||
print("\nDashboard generation and deployment complete!")
|
||||
|
||||
15
scripts/check_30.py
Normal file
15
scripts/check_30.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
c.connect('140.44.4.30', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
|
||||
print("Connected to 140.44.4.30 with snarf")
|
||||
stdin, stdout, _ = c.exec_command("hostname && ip a")
|
||||
print(stdout.read().decode())
|
||||
c.close()
|
||||
except Exception as e:
|
||||
print("Failed 140.44.4.30:", e)
|
||||
|
||||
9
scripts/check_all_pods.py
Normal file
9
scripts/check_all_pods.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get pods -A")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
@@ -1,17 +1,34 @@
|
||||
import os
|
||||
import sys
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
pkey = paramiko.Ed25519Key.from_private_key_file(os.path.expanduser('~/.ssh/id_ed25519_snarfnet'))
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.75', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
client.connect("140.44.4.11", username="root", pkey=pkey, timeout=10)
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
cmd = "curl -s http://140.44.4.21:80/main.105b4163.async.js | grep -o -E '\"/api/[a-zA-Z0-9_/]+\"' | sort -u | head -n 30"
|
||||
stdin, stdout, stderr = client.exec_command(cmd)
|
||||
print("API ROUTES ON 140.44.4.21:")
|
||||
print(stdout.read().decode())
|
||||
client.close()
|
||||
cid, _ = run("echo 'Katzir476!' | sudo -S docker ps -q -f name=peanut_peanut")
|
||||
|
||||
# Check readdir in src/app or src
|
||||
script = """
|
||||
const fs = require('fs');
|
||||
function walk(dir) {
|
||||
let res = [];
|
||||
for (let f of fs.readdirSync(dir)) {
|
||||
let p = dir + '/' + f;
|
||||
if (fs.statSync(p).isDirectory()) res.push(...walk(p));
|
||||
else if (f.endsWith('.ts') || f.endsWith('.js') || f.endsWith('.tsx')) res.push(p);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
console.log(walk('src/app/api'));
|
||||
"""
|
||||
run(f"cat << 'EOF' > /tmp/walk.js\n{script}\nEOF")
|
||||
out, err = run(f"echo 'Katzir476!' | sudo -S docker exec -i {cid} node < /tmp/walk.js")
|
||||
print(out)
|
||||
if err: print(err)
|
||||
|
||||
c.close()
|
||||
|
||||
9
scripts/check_argo_rev.py
Normal file
9
scripts/check_argo_rev.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.41', username='root', password='dietpi', timeout=10)
|
||||
_, o, e = c.exec_command("kubectl get app aerodrome-prometheus -n infrastructure")
|
||||
print("out:", o.read().decode().strip())
|
||||
print("err:", e.read().decode().strip())
|
||||
c.close()
|
||||
@@ -1,73 +1,8 @@
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcmdvY2QiLCJzdWIiOiJob21lcGFnZTphcGlLZXkiLCJuYmYiOjE3OTQ1MDIwNzgsImlhdCI6MTc5NDUwMjA3OCwianRpIjoiZjQ2NzU0NjMtODRhOS00MjI0LWI3ODItNmY5ODQxYWEwY2UxIn0.aHhrny2RBG-vAJAyXMvan0jw65jkP3bguqEduaG78O0"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
try:
|
||||
resp = requests.get("https://argo.snarfnet.net/api/v1/applications", headers=headers, verify=False, timeout=10)
|
||||
data = resp.json()
|
||||
items = data.get("items", [])
|
||||
print(f"Total Applications: {len(items)}\n")
|
||||
|
||||
out_of_sync = []
|
||||
synced = []
|
||||
|
||||
for app in items:
|
||||
name = app.get("metadata", {}).get("name")
|
||||
sync_status = app.get("status", {}).get("sync", {}).get("status")
|
||||
health_status = app.get("status", {}).get("health", {}).get("status")
|
||||
dest_server = app.get("spec", {}).get("destination", {}).get("server")
|
||||
dest_ns = app.get("spec", {}).get("destination", {}).get("namespace")
|
||||
repo = app.get("spec", {}).get("source", {}).get("repoURL")
|
||||
path = app.get("spec", {}).get("source", {}).get("path")
|
||||
|
||||
# Check operation or sync conditions/messages
|
||||
operation_state = app.get("status", {}).get("operationState", {})
|
||||
op_phase = operation_state.get("phase")
|
||||
op_msg = operation_state.get("message")
|
||||
conditions = app.get("status", {}).get("conditions", [])
|
||||
|
||||
info = {
|
||||
"name": name,
|
||||
"sync": sync_status,
|
||||
"health": health_status,
|
||||
"dest_ns": dest_ns,
|
||||
"dest_server": dest_server,
|
||||
"repo": repo,
|
||||
"path": path,
|
||||
"op_phase": op_phase,
|
||||
"op_msg": op_msg,
|
||||
"conditions": conditions
|
||||
}
|
||||
|
||||
if sync_status != "Synced" or health_status not in ["Healthy", None]:
|
||||
out_of_sync.append(info)
|
||||
else:
|
||||
synced.append(info)
|
||||
|
||||
print(f"=== OUT OF SYNC / DEGRADED APPS ({len(out_of_sync)}) ===")
|
||||
for a in out_of_sync:
|
||||
print(f"\nApp: {a['name']}")
|
||||
print(f" Sync: {a['sync']} | Health: {a['health']}")
|
||||
print(f" Dest: {a['dest_ns']} on {a['dest_server']}")
|
||||
print(f" Path: {a['path']}")
|
||||
if a['op_phase']:
|
||||
print(f" Op Phase: {a['op_phase']}")
|
||||
if a['op_msg']:
|
||||
print(f" Op Msg: {a['op_msg']}")
|
||||
if a['conditions']:
|
||||
print(f" Conditions: {a['conditions']}")
|
||||
|
||||
print(f"\n=== SYNCED APPS ({len(synced)}) ===")
|
||||
for a in synced:
|
||||
print(f" {a['name']}: Sync={a['sync']}, Health={a['health']}, NS={a['dest_ns']}")
|
||||
|
||||
except Exception as e:
|
||||
print("Error querying Argo CD API:", e)
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.41', username='root', password='dietpi', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get applications.argoproj.io -n infrastructure")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
@@ -1,27 +1,35 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import json
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=key_path, timeout=5)
|
||||
c.connect("140.44.4.41", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
cmd = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, authorization_flow_id FROM authentik_core_application;'"
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
print("Authentik Applications:")
|
||||
print(stdout.read().decode('utf-8'))
|
||||
def k8s(cmd):
|
||||
full_cmd = f"kubectl {cmd}"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
cmd2 = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, external_host FROM authentik_providers_proxy_proxyprovider;'"
|
||||
stdin, stdout, stderr = c.exec_command(cmd2)
|
||||
print("Authentik Proxy Providers:")
|
||||
print(stdout.read().decode('utf-8'))
|
||||
print("Authentik pods:")
|
||||
out, _ = k8s("get pods -A | grep -i authentik")
|
||||
print(out)
|
||||
|
||||
cmd3 = "k3s kubectl -n infra exec authentik-postgresql-0 -- psql -U authentik -d authentik -c 'SELECT id, name, client_id FROM authentik_providers_oauth2_oauth2provider;'"
|
||||
stdin, stdout, stderr = c.exec_command(cmd3)
|
||||
print("Authentik OAuth2/OIDC Providers:")
|
||||
print(stdout.read().decode('utf-8'))
|
||||
# Check Authentik pod name
|
||||
pod = ""
|
||||
for line in out.splitlines():
|
||||
if "server" in line:
|
||||
pod = line.split()[1]
|
||||
break
|
||||
|
||||
print("Authentik server pod:", pod)
|
||||
if pod:
|
||||
# Query Authentik applications and proxy providers via manage.py or python
|
||||
out, err = k8s(f"exec -n authentik {pod} -- ak list_apps 2>/dev/null || kubectl exec -n authentik {pod} -- python -m authentik.manage shell -c 'from authentik.core.models import Application; print([(a.name, a.slug) for a in Application.objects.all()])'")
|
||||
print("Applications:", out, err)
|
||||
|
||||
# Query Authentik proxy providers
|
||||
out, err = k8s(f"exec -n authentik {pod} -- python -m authentik.manage shell -c 'from authentik.providers.proxy.models import ProxyProvider; print([(p.name, p.external_host) for p in ProxyProvider.objects.all()])'")
|
||||
print("Proxy Providers:", out, err)
|
||||
|
||||
c.close()
|
||||
|
||||
|
||||
35
scripts/check_authentik_db.py
Normal file
35
scripts/check_authentik_db.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
_, o, _ = c.exec_command("kubectl exec -n infra authentik-postgresql-0 -- cat /opt/bitnami/postgresql/secrets/password")
|
||||
pw = o.read().decode().strip()
|
||||
|
||||
def psql(query):
|
||||
cmd = f'kubectl exec -n infra authentik-postgresql-0 -- sh -c "PGPASSWORD={pw} psql -U authentik -d authentik -c \\"{query}\\""'
|
||||
_, o, e = c.exec_command(cmd)
|
||||
res = o.read().decode().strip()
|
||||
err = e.read().decode().strip()
|
||||
if err and not res:
|
||||
return f"[ERR: {err}]"
|
||||
return res
|
||||
|
||||
print("--- APPLICATIONS ---")
|
||||
print(psql("SELECT name, slug FROM authentik_core_application;"))
|
||||
|
||||
print("\n--- PROXY PROVIDERS ---")
|
||||
print(psql("SELECT name, external_host, mode FROM authentik_providers_proxy_proxyprovider;"))
|
||||
|
||||
print("\n--- EXPRESSION POLICIES ---")
|
||||
print(psql("SELECT name, expression FROM authentik_policies_expression_expressionpolicy;"))
|
||||
|
||||
print("\n--- POLICY BINDINGS ---")
|
||||
print(psql("SELECT target_id, enabled FROM authentik_policies_policybinding;"))
|
||||
|
||||
print("\n--- AUTHENTIK SETTINGS / SESSIONS ---")
|
||||
print(psql("SELECT key, value FROM authentik_tenants_tenant;"))
|
||||
|
||||
c.close()
|
||||
32
scripts/check_authentik_file.py
Normal file
32
scripts/check_authentik_file.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
_, o, _ = c.exec_command("kubectl exec -n infra authentik-postgresql-0 -- cat /opt/bitnami/postgresql/secrets/password")
|
||||
pw = o.read().decode().strip()
|
||||
|
||||
sql = """
|
||||
\\echo '=== APPS ==='
|
||||
SELECT name, slug FROM authentik_core_application;
|
||||
\\echo '=== PROVIDERS ==='
|
||||
SELECT name, external_host, mode FROM authentik_providers_proxy_proxyprovider;
|
||||
\\echo '=== POLICIES ==='
|
||||
SELECT name, expression FROM authentik_policies_expression_expressionpolicy;
|
||||
"""
|
||||
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/query.sql")
|
||||
stdin.write(sql)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
c.exec_command("kubectl cp /tmp/query.sql infra/authentik-postgresql-0:/tmp/query.sql")
|
||||
cmd = f"kubectl exec -n infra authentik-postgresql-0 -- env PGPASSWORD='{pw}' psql -U authentik -d authentik -f /tmp/query.sql"
|
||||
_, o, e = c.exec_command(cmd)
|
||||
print(o.read().decode())
|
||||
print("ERR:", e.read().decode())
|
||||
|
||||
c.close()
|
||||
|
||||
46
scripts/check_authentik_fixed.py
Normal file
46
scripts/check_authentik_fixed.py
Normal file
@@ -0,0 +1,46 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"kubectl {cmd}"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
pod = "authentik-server-cd7d9b949-vfl2m"
|
||||
|
||||
script = '''
|
||||
from authentik.core.models import Application
|
||||
from authentik.providers.proxy.models import ProxyProvider
|
||||
from authentik.policies.expression.models import ExpressionPolicy
|
||||
|
||||
print("=== APPLICATIONS ===")
|
||||
for app in Application.objects.all():
|
||||
print(f"{app.name} (slug: {app.slug}) | Provider: {app.provider}")
|
||||
|
||||
print("=== PROXY PROVIDERS ===")
|
||||
for p in ProxyProvider.objects.all():
|
||||
print(f"{p.name} | Host: {p.external_host} | Mode: {p.mode}")
|
||||
|
||||
print("=== EXPRESSION POLICIES ===")
|
||||
for ep in ExpressionPolicy.objects.all():
|
||||
print(f"Policy: {ep.name}")
|
||||
print(ep.expression)
|
||||
'''
|
||||
|
||||
# Write script to /tmp/ak_query.py on 140.44.4.11
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/ak_query.py")
|
||||
stdin.write(script)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
# Copy into pod and execute
|
||||
k8s(f"cp /tmp/ak_query.py infra/{pod}:/tmp/ak_query.py -c server")
|
||||
out, err = k8s(f"exec -n infra {pod} -c server -- /lifecycle/ak shell < /tmp/ak_query.py")
|
||||
print("Authentik query output:\n", out)
|
||||
if err: print("ERR:\n", err)
|
||||
|
||||
c.close()
|
||||
49
scripts/check_authentik_policies.py
Normal file
49
scripts/check_authentik_policies.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"kubectl {cmd}"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
pod = "authentik-server-cd7d9b949-vfl2m"
|
||||
|
||||
# Inspect applications in Authentik
|
||||
py_query = """
|
||||
python3 - << 'PYEOF'
|
||||
from authentik.core.models import Application
|
||||
from authentik.providers.proxy.models import ProxyProvider
|
||||
from authentik.policies.models import PolicyBindingModel, Policy
|
||||
|
||||
print("=== APPLICATIONS ===")
|
||||
for app in Application.objects.all():
|
||||
print(f"App: {app.name} (slug: {app.slug})")
|
||||
print(f" Provider: {app.provider}")
|
||||
|
||||
print("\n=== PROXY PROVIDERS ===")
|
||||
for p in ProxyProvider.objects.all():
|
||||
print(f"Provider: {p.name}")
|
||||
print(f" External host: {p.external_host}")
|
||||
print(f" Internal host: {p.internal_host}")
|
||||
print(f" Mode: {p.mode}")
|
||||
|
||||
print("\n=== POLICIES ===")
|
||||
for pol in Policy.objects.all():
|
||||
print(f"Policy: {pol.name} ({pol.__class__.__name__})")
|
||||
|
||||
print("\n=== POLICY BINDINGS ===")
|
||||
for pb in PolicyBindingModel.objects.all():
|
||||
print(f"Binding: {pb.target} -> {pb.policy} (enabled={pb.enabled}, order={pb.order})")
|
||||
PYEOF
|
||||
"""
|
||||
|
||||
out, err = k8s(f"exec -n infra {pod} -c authentik-server -- {py_query}")
|
||||
print(out)
|
||||
if err: print("ERR:", err)
|
||||
|
||||
c.close()
|
||||
|
||||
28
scripts/check_authentik_policies_sql.py
Normal file
28
scripts/check_authentik_policies_sql.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
_, o, _ = c.exec_command("kubectl exec -n infra authentik-postgresql-0 -- cat /opt/bitnami/postgresql/secrets/password")
|
||||
pw = o.read().decode().strip()
|
||||
|
||||
sql = """
|
||||
SELECT p.name, ep.expression
|
||||
FROM authentik_policies_expression_expressionpolicy ep
|
||||
JOIN authentik_policies_policy p ON ep.policy_ptr_id = p.policy_uuid;
|
||||
"""
|
||||
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/query_policies.sql")
|
||||
stdin.write(sql)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
c.exec_command("kubectl cp /tmp/query_policies.sql infra/authentik-postgresql-0:/tmp/query_policies.sql")
|
||||
cmd = f"kubectl exec -n infra authentik-postgresql-0 -- env PGPASSWORD='{pw}' psql -U authentik -d authentik -f /tmp/query_policies.sql"
|
||||
_, o, e = c.exec_command(cmd)
|
||||
print(o.read().decode())
|
||||
|
||||
c.close()
|
||||
|
||||
28
scripts/check_authentik_stdin.py
Normal file
28
scripts/check_authentik_stdin.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.11", username="root", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
_, o, _ = c.exec_command("kubectl exec -n infra authentik-postgresql-0 -- cat /opt/bitnami/postgresql/secrets/password")
|
||||
pw = o.read().decode().strip()
|
||||
|
||||
def run_sql(sql):
|
||||
cmd = f"kubectl exec -i -n infra authentik-postgresql-0 -- /bin/bash -c 'export PGPASSWORD={pw}; psql -U authentik -d authentik'"
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
stdin.write(sql + "\n")
|
||||
stdin.close()
|
||||
return stdout.read().decode().strip()
|
||||
|
||||
print("--- APPLICATIONS ---")
|
||||
print(run_sql("SELECT name, slug FROM authentik_core_application;"))
|
||||
|
||||
print("\n--- PROXY PROVIDERS ---")
|
||||
print(run_sql("SELECT name, external_host, mode FROM authentik_providers_proxy_proxyprovider;"))
|
||||
|
||||
print("\n--- EXPRESSION POLICIES ---")
|
||||
print(run_sql("SELECT name, expression FROM authentik_policies_expression_expressionpolicy;"))
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_blockdevices.py
Normal file
22
scripts/check_blockdevices.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Harvester Block Devices:")
|
||||
out, _ = k8s("kubectl get blockdevices.harvesterhci.io -A -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,NODE:.spec.nodeName,DEV:.status.devicePath,FS:.status.fileSystem.type,MOUNT:.status.fileSystem.mountPoint,PROVISIONED:.status.provisioned")
|
||||
print(out)
|
||||
|
||||
print("\nHarvester Node Disk status:")
|
||||
out, _ = k8s("kubectl get nodes.harvesterhci.io -A -o yaml | grep -E 'nodeName|diskStatus|mountPath|provisioned'")
|
||||
print(out[:1000])
|
||||
|
||||
c.close()
|
||||
|
||||
10
scripts/check_cadvisor_ps.py
Normal file
10
scripts/check_cadvisor_ps.py
Normal file
@@ -0,0 +1,10 @@
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("echo 'Katzir476!' | sudo -S docker stack ps monitoring")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
23
scripts/check_cm_live.py
Normal file
23
scripts/check_cm_live.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
_, o, _ = c.exec_command("kubectl get configmap prometheus-config -o json")
|
||||
import json
|
||||
data = json.loads(o.read().decode().strip())
|
||||
prom_yml = data.get("data", {}).get("prometheus.yml", "")
|
||||
print("Length of prometheus.yml in CM:", len(prom_yml))
|
||||
print("Contains shipyard:", "shipyard" in prom_yml)
|
||||
if "shipyard" in prom_yml:
|
||||
for line in prom_yml.splitlines():
|
||||
if "shipyard" in line:
|
||||
print(" ", line)
|
||||
else:
|
||||
print("Last 20 lines of prometheus.yml:")
|
||||
for line in prom_yml.splitlines()[-20:]:
|
||||
print(" ", line)
|
||||
|
||||
c.close()
|
||||
|
||||
11
scripts/check_cm_meta.py
Normal file
11
scripts/check_cm_meta.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get configmap prometheus-config -o json")
|
||||
data = json.loads(o.read().decode().strip())
|
||||
print(json.dumps(data.get("metadata"), indent=2))
|
||||
c.close()
|
||||
|
||||
11
scripts/check_cm_now.py
Normal file
11
scripts/check_cm_now.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get configmap prometheus-config -o yaml")
|
||||
yaml_str = o.read().decode().strip()
|
||||
print("Length of CM:", len(yaml_str))
|
||||
print("Contains shipyard:", "shipyard" in yaml_str)
|
||||
c.close()
|
||||
|
||||
9
scripts/check_cm_tmp.py
Normal file
9
scripts/check_cm_tmp.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("cat /tmp/prometheus_cm.yaml | grep -A 25 'job_name: node-exporter'")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
25
scripts/check_compute_fw.py
Normal file
25
scripts/check_compute_fw.py
Normal file
@@ -0,0 +1,25 @@
|
||||
import requests
|
||||
import urllib3
|
||||
import json
|
||||
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
s = requests.Session()
|
||||
s.verify = False
|
||||
|
||||
r = s.post('https://14.44.4.1/api/auth/login', json={'username': 'snarf', 'password': 'Katzir476!!!'})
|
||||
csrf = r.headers.get('x-csrf-token') or r.cookies.get('csrf_token')
|
||||
headers = {'X-CSRF-Token': csrf} if csrf else {}
|
||||
|
||||
res = s.get('https://14.44.4.1/proxy/network/v2/api/site/default/firewall-policies', headers=headers)
|
||||
policies = res.json()
|
||||
compute_net_id = "1748bb1d-f674-4dcd-85d0-f58109a02e77"
|
||||
|
||||
for p in policies:
|
||||
src_nets = p.get('source', {}).get('network_ids', []) or []
|
||||
dst_nets = p.get('destination', {}).get('network_ids', []) or []
|
||||
if compute_net_id in src_nets or compute_net_id in dst_nets:
|
||||
print(f"Policy: {p.get('name')} (Index {p.get('index')}) | Action: {p.get('action')} | Type: {p.get('connection_state_type')}")
|
||||
print(f" Src: {p.get('source')}")
|
||||
print(f" Dst: {p.get('destination')}")
|
||||
|
||||
31
scripts/check_crd_salvage.py
Normal file
31
scripts/check_crd_salvage.py
Normal file
@@ -0,0 +1,31 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Replica spec fields:")
|
||||
out, _ = k8s("kubectl explain replica.longhorn.io.spec")
|
||||
print(out)
|
||||
|
||||
print("\nVolume spec fields:")
|
||||
out, _ = k8s("kubectl explain volume.longhorn.io.spec")
|
||||
print(out)
|
||||
|
||||
print("\nEngine spec fields:")
|
||||
out, _ = k8s("kubectl explain engine.longhorn.io.spec")
|
||||
print(out)
|
||||
|
||||
# Check engine CR directly
|
||||
out, _ = k8s("kubectl get engines.longhorn.io -n longhorn-system pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495-e-0 -o yaml")
|
||||
print("\nEngine YAML for pvc-4da99c48:")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
30
scripts/check_cross_vlan.py
Normal file
30
scripts/check_cross_vlan.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("Ping 14.4.4.20 (HomeAssistant):")
|
||||
out, _ = run("ping -c 2 -W 2 14.4.4.20")
|
||||
print(out)
|
||||
|
||||
print("Ping 14.4.4.15:")
|
||||
out, _ = run("ping -c 2 -W 2 14.4.4.15")
|
||||
print(out)
|
||||
|
||||
print("Ping 14.4.4.44 (nutserver):")
|
||||
out, _ = run("ping -c 2 -W 2 14.4.4.44")
|
||||
print(out)
|
||||
|
||||
print("Traceroute to 14.4.4.44:")
|
||||
out, _ = run("traceroute 14.4.4.44 || tracepath 14.4.4.44")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
33
scripts/check_current_harvester_tags.py
Normal file
33
scripts/check_current_harvester_tags.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run_sql(sql):
|
||||
pod_cmd = "kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'"
|
||||
_, o, _ = c.exec_command(pod_cmd)
|
||||
pod = o.read().decode().strip()
|
||||
cmd = f"kubectl exec -i -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root"
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
stdin.write(sql)
|
||||
stdin.close()
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- Monitor tags for 57, 94, 98-105 ---")
|
||||
query = """
|
||||
USE kuma;
|
||||
SELECT m.id, m.name, t.name as tag_name
|
||||
FROM monitor m
|
||||
LEFT JOIN monitor_tag mt ON m.id = mt.monitor_id
|
||||
LEFT JOIN tag t ON mt.tag_id = t.id
|
||||
WHERE m.id IN (57, 94, 98, 99, 100, 101, 102, 103, 104, 105)
|
||||
ORDER BY m.id, t.id;
|
||||
"""
|
||||
out, _ = run_sql(query)
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
13
scripts/check_deploy_vols.py
Normal file
13
scripts/check_deploy_vols.py
Normal file
@@ -0,0 +1,13 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get deploy prometheus -o json")
|
||||
data = json.loads(o.read().decode().strip())
|
||||
volumes = data.get("spec", {}).get("template", {}).get("spec", {}).get("volumes", [])
|
||||
for v in volumes:
|
||||
print(v)
|
||||
c.close()
|
||||
|
||||
23
scripts/check_disk_contents.py
Normal file
23
scripts/check_disk_contents.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.81", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def run_cmd(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Mount /dev/sdf to /mnt and check contents:")
|
||||
out, err = run_cmd("mkdir -p /mnt/test && mount /dev/sdf /mnt/test && ls -la /mnt/test")
|
||||
print("sdf:", out, err)
|
||||
|
||||
# Check all devices sdb through sdh
|
||||
for d in ["sdb", "sdc", "sdd", "sde", "sdf", "sdg", "sdh"]:
|
||||
out, _ = run_cmd(f"mkdir -p /mnt/{d} && mount /dev/{d} /mnt/{d} 2>/dev/null && ls -la /mnt/{d} | grep -E 'longhorn|replicas'")
|
||||
print(f"{d}: {out}")
|
||||
|
||||
c.close()
|
||||
|
||||
23
scripts/check_disk_contents2.py
Normal file
23
scripts/check_disk_contents2.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.81", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def run_cmd(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Mount /dev/sdf to /tmp/test:")
|
||||
out, err = run_cmd("mkdir -p /tmp/test && mount /dev/sdf /tmp/test && ls -la /tmp/test; umount /tmp/test")
|
||||
print("sdf:", out, err)
|
||||
|
||||
# Check all devices on Node 1:
|
||||
for d in ["sdb", "sdc", "sdd", "sde", "sdf", "sdg", "sdh"]:
|
||||
out, _ = run_cmd(f"mkdir -p /tmp/{d} && mount /dev/{d} /tmp/{d} 2>/dev/null && ls -la /tmp/{d}; umount /tmp/{d} 2>/dev/null")
|
||||
print(f"--- {d} ---:\n{out}")
|
||||
|
||||
c.close()
|
||||
|
||||
35
scripts/check_disks_mount.py
Normal file
35
scripts/check_disks_mount.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("=== NODES.LONGHORN.IO DISK STATUS ===")
|
||||
out, _ = k8s("kubectl get nodes.longhorn.io -n longhorn-system -o yaml | grep -E 'diskStatus|diskUUID|ready|schedulable|storageAvailable|storageMaximum|path:'")
|
||||
print(out[:1500])
|
||||
|
||||
print("\n=== LSBLK & MOUNTS ON NODE 01 ===")
|
||||
out, _ = k8s("lsblk -f; echo '--- DF ---'; df -h | grep -E 'Filesystem|extra-disks|harvester'")
|
||||
print(out)
|
||||
|
||||
# Check all 4 nodes for extra-disks mount
|
||||
for ip in ["140.44.4.81", "140.44.4.82", "140.44.4.83", "140.44.4.84"]:
|
||||
print(f"\n--- Node {ip} ---")
|
||||
try:
|
||||
nc = paramiko.SSHClient()
|
||||
nc.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
nc.connect(ip, username="rancher", key_filename=SSH_KEY, timeout=5)
|
||||
_, o, _ = nc.exec_command("echo Katzir476! | sudo -S lsblk | grep -E 'sd|nvme'; df -h | grep extra")
|
||||
print(o.read().decode().strip())
|
||||
nc.close()
|
||||
except Exception as e:
|
||||
print("Error connecting:", e)
|
||||
|
||||
c.close()
|
||||
|
||||
27
scripts/check_dns_heimdall.py
Normal file
27
scripts/check_dns_heimdall.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import ssl
|
||||
import json
|
||||
import http.client
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def do_req(ip, host_header, path):
|
||||
conn = http.client.HTTPSConnection(ip, 443, context=ctx, timeout=8)
|
||||
conn.request("GET", path, headers={"Host": host_header})
|
||||
resp = conn.getresponse()
|
||||
data = resp.read().decode('utf-8')
|
||||
conn.close()
|
||||
return json.loads(data)
|
||||
|
||||
try:
|
||||
token = do_req("140.44.4.62", "dns2.snarfnet.net", "/api/user/login?user=admin&pass=snarfnetadmin&includeInfo=true")['token']
|
||||
rec_heimdall = do_req("140.44.4.62", "dns2.snarfnet.net", f"/api/zones/records/get?token={token}&zone=snarfnet.net&domain=heimdall.snarfnet.net")
|
||||
print("DNS2 heimdall:", rec_heimdall)
|
||||
rec_portal = do_req("140.44.4.62", "dns2.snarfnet.net", f"/api/zones/records/get?token={token}&zone=snarfnet.net&domain=portal.snarfnet.net")
|
||||
print("DNS2 portal:", rec_portal)
|
||||
except Exception as e:
|
||||
print("DNS check err:", e)
|
||||
|
||||
19
scripts/check_docker_sudo.py
Normal file
19
scripts/check_docker_sudo.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- echo 'Katzir476!' | sudo -S docker service ls ---")
|
||||
out, err = run("echo 'Katzir476!' | sudo -S docker service ls")
|
||||
print("OUT:\n", out)
|
||||
print("ERR:\n", err)
|
||||
|
||||
c.close()
|
||||
|
||||
9
scripts/check_etc_prom.py
Normal file
9
scripts/check_etc_prom.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl exec deploy/prometheus -- ls -la /etc/prometheus")
|
||||
print("ls -la /etc/prometheus:\n", o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
30
scripts/check_fstab_mounts.py
Normal file
30
scripts/check_fstab_mounts.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.81", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def run_cmd(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("--- /etc/fstab on Node 1 ---")
|
||||
out, _ = run_cmd("cat /etc/fstab")
|
||||
print(out)
|
||||
|
||||
print("\n--- Systemd mount units on Node 1 ---")
|
||||
out, _ = run_cmd("systemctl list-unit-files | grep -i var-lib-harvester")
|
||||
print(out)
|
||||
|
||||
print("\n--- Blkid on Node 1 ---")
|
||||
out, _ = run_cmd("blkid")
|
||||
print(out)
|
||||
|
||||
print("\n--- Any harvester disk mount script or cloud-init ---")
|
||||
out, _ = run_cmd("ls -la /oem; cat /oem/99_custom.yaml 2>/dev/null || cat /oem/harvester.config 2>/dev/null")
|
||||
print(out[:1000])
|
||||
|
||||
c.close()
|
||||
|
||||
28
scripts/check_gotify_discord.py
Normal file
28
scripts/check_gotify_discord.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', password='Katzir476!', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("=== GOTIFY DISCORD BRIDGE LOGS ===")
|
||||
cid, _ = run("docker ps -q -f name=gotify-discord-bridge")
|
||||
if cid:
|
||||
logs, _ = run(f"docker logs {cid} --tail=100")
|
||||
print(logs)
|
||||
else:
|
||||
print("Bridge container not found!")
|
||||
|
||||
print("\n=== GOTIFY SERVER RECENT MESSAGES ===")
|
||||
# Check messages inside Gotify DB or logs
|
||||
gid, _ = run("docker ps -q -f name=gotify_gotify")
|
||||
if gid:
|
||||
logs, _ = run(f"docker logs {gid} --tail=50")
|
||||
print(logs)
|
||||
|
||||
c.close()
|
||||
|
||||
14
scripts/check_grafana_ds.py
Normal file
14
scripts/check_grafana_ds.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
# Let's query Prometheus datasource from Grafana or directly from 140.44.4.11/140.44.4.34
|
||||
url = "https://grafana.snarfnet.net/api/datasources"
|
||||
resp = requests.get(url, auth=auth, verify=False, timeout=10)
|
||||
datasources = resp.json()
|
||||
print("Datasources:")
|
||||
for ds in datasources:
|
||||
print(f"ID: {ds['id']} | Name: {ds['name']} | Type: {ds['type']} | URL: {ds['url']}")
|
||||
|
||||
17
scripts/check_ha.py
Normal file
17
scripts/check_ha.py
Normal file
@@ -0,0 +1,17 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
c.connect('14.4.4.20', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
|
||||
print("Connected to 14.4.4.20")
|
||||
except Exception as e:
|
||||
print("Failed key:", e)
|
||||
try:
|
||||
c.connect('14.4.4.20', username='root', password='dietpi', timeout=5)
|
||||
print("Connected with root/dietpi")
|
||||
except Exception as e2:
|
||||
print("Failed root/dietpi:", e2)
|
||||
|
||||
17
scripts/check_ha_ups.py
Normal file
17
scripts/check_ha_ups.py
Normal file
@@ -0,0 +1,17 @@
|
||||
import urllib.request
|
||||
import json
|
||||
|
||||
HA_URL = "http://14.4.4.20:8123"
|
||||
HA_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkZWNlMWI5ZjJhZDQ0MTRmODU4NGYzN2I1YjgwZDQxMiIsImlhdCI6MTc5MzQ1MjQ2MywiZXhwIjoyMTA4ODEyNDYzfQ.JznpOqsCSnHz6aTVa8K9pQCz1816DC8SrmS1xn1Sm2k"
|
||||
|
||||
req = urllib.request.Request(f"{HA_URL}/api/states", headers={"Authorization": f"Bearer {HA_TOKEN}", "Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=5) as r:
|
||||
states = json.loads(r.read().decode())
|
||||
ups_states = [s for s in states if "ups" in s["entity_id"] or "server_closet" in s["entity_id"] or "nut" in s["entity_id"]]
|
||||
print(f"Found {len(ups_states)} UPS entities in Home Assistant:")
|
||||
for s in ups_states:
|
||||
print(f" {s['entity_id']}: {s['state']} {s.get('attributes', {}).get('unit_of_measurement', '')}")
|
||||
except Exception as e:
|
||||
print("HA err:", e)
|
||||
|
||||
49
scripts/check_heimdall_db_deep.py
Normal file
49
scripts/check_heimdall_db_deep.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
py_script = """python3 - << 'PYEOF'
|
||||
import sqlite3
|
||||
import shutil
|
||||
import time
|
||||
|
||||
db_path = '/var/lib/docker/volumes/heimdall_heimdall_config/_data/www/app.sqlite'
|
||||
|
||||
# Make a fresh backup
|
||||
shutil.copyfile(db_path, f"{db_path}.verified_backup_{int(time.time())}")
|
||||
print("Backup created at verified_backup!")
|
||||
|
||||
con = sqlite3.connect(db_path)
|
||||
cur = con.cursor()
|
||||
|
||||
print("=== USERS ===")
|
||||
for r in cur.execute("SELECT id, username, email, public_front FROM users").fetchall():
|
||||
print("User:", r)
|
||||
|
||||
print("\n=== ITEMS ===")
|
||||
rows = cur.execute("SELECT id, title, url, pinned, user_id, apporder FROM items").fetchall()
|
||||
print(f"Total items in DB: {len(rows)}")
|
||||
for r in rows:
|
||||
print("Item:", r)
|
||||
|
||||
print("\n=== SETTINGS ===")
|
||||
for r in cur.execute("SELECT * FROM settings").fetchall():
|
||||
print("Setting:", r)
|
||||
|
||||
PYEOF
|
||||
"""
|
||||
|
||||
_, o, e = c.exec_command(f"echo Katzir476! | sudo -S {py_script}")
|
||||
print(o.read().decode())
|
||||
print(e.read().decode())
|
||||
|
||||
# Check volume mount persistence
|
||||
_, o, _ = c.exec_command("echo Katzir476! | sudo -S ls -la /var/lib/docker/volumes/heimdall_heimdall_config/_data/www/")
|
||||
print("Files in Heimdall www volume:\n", o.read().decode())
|
||||
|
||||
c.close()
|
||||
|
||||
47
scripts/check_heimdall_db_fixed.py
Normal file
47
scripts/check_heimdall_db_fixed.py
Normal file
@@ -0,0 +1,47 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
py_code = '''
|
||||
import sqlite3
|
||||
import shutil
|
||||
import time
|
||||
|
||||
db_path = '/var/lib/docker/volumes/heimdall_heimdall_config/_data/www/app.sqlite'
|
||||
backup_name = db_path + '.backup_' + str(int(time.time()))
|
||||
shutil.copyfile(db_path, backup_name)
|
||||
print('Backup created:', backup_name)
|
||||
|
||||
con = sqlite3.connect(db_path)
|
||||
cur = con.cursor()
|
||||
|
||||
print('=== USERS ===')
|
||||
for r in cur.execute('SELECT id, username, email, public_front FROM users').fetchall():
|
||||
print('User:', r)
|
||||
|
||||
print('\\n=== ITEMS ===')
|
||||
rows = cur.execute('SELECT id, title, url, pinned, user_id, apporder FROM items').fetchall()
|
||||
print('Total items in DB:', len(rows))
|
||||
for r in rows:
|
||||
print('Item:', r)
|
||||
|
||||
print('\\n=== SETTINGS ===')
|
||||
for r in cur.execute('SELECT * FROM settings').fetchall():
|
||||
print('Setting:', r)
|
||||
'''
|
||||
|
||||
# Write script to /tmp/inspect_heimdall.py on remote host
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/inspect_heimdall.py")
|
||||
stdin.write(py_code)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
# Execute script with sudo
|
||||
_, o, e = c.exec_command("echo Katzir476! | sudo -S python3 /tmp/inspect_heimdall.py")
|
||||
print(o.read().decode())
|
||||
print(e.read().decode())
|
||||
|
||||
c.close()
|
||||
|
||||
19
scripts/check_heimdall_final.py
Normal file
19
scripts/check_heimdall_final.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
cmd = """echo 'Katzir476!' | sudo -S python3 -c "
|
||||
import sqlite3
|
||||
con = sqlite3.connect('/var/lib/docker-data/heimdall/config/www/app.sqlite')
|
||||
cur = con.cursor()
|
||||
print('Users:', cur.execute('SELECT id, username, public_front FROM users').fetchall())
|
||||
print('Pinned Items:', cur.execute('SELECT count(*) FROM items WHERE pinned=1').fetchone()[0])
|
||||
con.close()
|
||||
" """
|
||||
_, o, _ = c.exec_command(cmd)
|
||||
print("Output:\n", o.read().decode().strip())
|
||||
c.close()
|
||||
19
scripts/check_heimdall_mount_real.py
Normal file
19
scripts/check_heimdall_mount_real.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
out, _ = run("docker inspect $(docker ps -q -f name=heimdall) | grep -A 10 -i mounts")
|
||||
print("Container Mounts:\n", out)
|
||||
|
||||
out, _ = run("docker service inspect heimdall_heimdall --pretty")
|
||||
print("\nService inspect pretty:\n", out)
|
||||
|
||||
c.close()
|
||||
|
||||
21
scripts/check_heimdall_volume.py
Normal file
21
scripts/check_heimdall_volume.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Docker Volume Inspect:")
|
||||
out, _ = run("docker volume inspect heimdall_heimdall_config")
|
||||
print(out)
|
||||
|
||||
print("\nHeimdall service constraints:")
|
||||
out, _ = run("docker service inspect heimdall_heimdall --format '{{json .Spec.TaskTemplate.Placement}}'")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
27
scripts/check_idrac4_details.py
Normal file
27
scripts/check_idrac4_details.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import urllib.request
|
||||
import json
|
||||
import ssl
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def zabbix_api(method, params, auth=None):
|
||||
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
|
||||
if auth: payload["auth"] = auth
|
||||
req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php",
|
||||
data=json.dumps(payload).encode('utf-8'),
|
||||
headers={"Content-Type": "application/json-rpc"})
|
||||
resp = urllib.request.urlopen(req, context=ctx, timeout=10)
|
||||
return json.loads(resp.read().decode('utf-8'))
|
||||
|
||||
auth = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"}).get("result")
|
||||
|
||||
h = zabbix_api("host.get", {
|
||||
"filter": {"host": ["idrac-spacedock4"]},
|
||||
"selectInterfaces": "extend",
|
||||
"selectGroups": "extend",
|
||||
"selectParentTemplates": "extend"
|
||||
}, auth=auth).get("result", [])
|
||||
print(json.dumps(h, indent=2))
|
||||
|
||||
23
scripts/check_instance_manager.py
Normal file
23
scripts/check_instance_manager.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Instance Manager pods:")
|
||||
out, _ = k8s("kubectl get pods -n longhorn-system -l longhorn.io/component=instance-manager -o wide")
|
||||
print(out)
|
||||
|
||||
# Check Longhorn volume CR
|
||||
out, _ = k8s("kubectl get volume.longhorn.io pvc-4da99c48-f52e-4adf-9b4c-16e8878eb495 -n longhorn-system -o yaml")
|
||||
print("\nVolume YAML for pvc-4da99c48:")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
23
scripts/check_items_schema.py
Normal file
23
scripts/check_items_schema.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
cmd = """echo 'Katzir476!' | sudo -S python3 -c "
|
||||
import sqlite3
|
||||
con = sqlite3.connect('/var/lib/docker-data/heimdall/config/www/app.sqlite')
|
||||
cur = con.cursor()
|
||||
print('Item columns:', [d[0] for d in cur.execute('SELECT * FROM items LIMIT 1').description])
|
||||
rows = cur.execute('SELECT id, title, pinned, user_id FROM items LIMIT 5').fetchall()
|
||||
print('Sample items:', rows)
|
||||
print('Total items count:', cur.execute('SELECT count(*) FROM items').fetchone()[0])
|
||||
print('Distinct pinned values:', cur.execute('SELECT DISTINCT pinned FROM items').fetchall())
|
||||
con.close()
|
||||
" """
|
||||
_, o, _ = c.exec_command(cmd)
|
||||
print("Output:\n", o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
21
scripts/check_kuma_monitor_schema.py
Normal file
21
scripts/check_kuma_monitor_schema.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip()
|
||||
|
||||
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
||||
res = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE monitor;" """)
|
||||
print("Monitor Columns:\n", res)
|
||||
|
||||
res2 = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM monitor WHERE id=94;" """)
|
||||
print("\nMonitor 94:\n", res2)
|
||||
|
||||
c.close()
|
||||
|
||||
21
scripts/check_kuma_notifs.py
Normal file
21
scripts/check_kuma_notifs.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip()
|
||||
|
||||
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
||||
res = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; DESCRIBE notification; SELECT * FROM notification;" """)
|
||||
print(res)
|
||||
|
||||
notif_mon = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM monitor_notification WHERE monitor_id IN (57, 94);" """)
|
||||
print("\nMonitor Notifications for 57, 94:")
|
||||
print(notif_mon)
|
||||
|
||||
c.close()
|
||||
18
scripts/check_kuma_ping_format.py
Normal file
18
scripts/check_kuma_ping_format.py
Normal file
@@ -0,0 +1,18 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip()
|
||||
|
||||
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
||||
res = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT * FROM monitor WHERE type='ping' LIMIT 2 \G" """)
|
||||
print(res)
|
||||
|
||||
c.close()
|
||||
|
||||
26
scripts/check_kuma_spacedock.py
Normal file
26
scripts/check_kuma_spacedock.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip()
|
||||
|
||||
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
||||
print("Kuma Pod:", pod)
|
||||
|
||||
# Check monitors related to harvester or spacedock
|
||||
sql = "USE kuma; SELECT id, name, type, hostname, port, url, active FROM monitor WHERE name LIKE '%harvester%' OR name LIKE '%spacedock%' OR hostname LIKE '%140.44.4.8%' OR hostname LIKE '%140.44.44.12%';"
|
||||
res = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "{sql}" """)
|
||||
print(res)
|
||||
|
||||
# Also check notifications
|
||||
notifs = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, is_default, active FROM notification;" """)
|
||||
print("\nNotifications:")
|
||||
print(notifs)
|
||||
|
||||
c.close()
|
||||
32
scripts/check_kuma_tags.py
Normal file
32
scripts/check_kuma_tags.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run_sql(sql):
|
||||
pod_cmd = "kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'"
|
||||
_, o, _ = c.exec_command(pod_cmd)
|
||||
pod = o.read().decode().strip()
|
||||
cmd = f"kubectl exec -i -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root"
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
stdin.write(sql)
|
||||
stdin.close()
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- Tables ---")
|
||||
out, _ = run_sql("USE kuma; SHOW TABLES LIKE '%tag%';")
|
||||
print(out)
|
||||
|
||||
print("--- Tags ---")
|
||||
out, _ = run_sql("USE kuma; SELECT * FROM tag;")
|
||||
print(out)
|
||||
|
||||
print("--- Monitor Tags sample ---")
|
||||
out, _ = run_sql("USE kuma; SELECT * FROM monitor_tag LIMIT 15;")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
21
scripts/check_kuma_types.py
Normal file
21
scripts/check_kuma_types.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip()
|
||||
|
||||
pod = run("kubectl get pod -n monitoring -l app.kubernetes.io/name=uptime-kuma -o jsonpath='{.items[0].metadata.name}'")
|
||||
res = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT DISTINCT type FROM monitor;" """)
|
||||
print("Types:", res)
|
||||
|
||||
res2 = run(f"""kubectl exec -n monitoring {pod} -- mariadb --socket=/app/data/run/mariadb.sock -u root -e "USE kuma; SELECT id, name, type, hostname, port, interval FROM monitor LIMIT 5;" """)
|
||||
print("Sample monitors:\n", res2)
|
||||
|
||||
c.close()
|
||||
|
||||
30
scripts/check_lh_disks.py
Normal file
30
scripts/check_lh_disks.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Nodes in Longhorn (spec.disks vs status.diskStatus):")
|
||||
out, _ = k8s("kubectl get nodes.longhorn.io -n longhorn-system -o json")
|
||||
data = json.loads(out)
|
||||
for node in data.get("items", []):
|
||||
name = node["metadata"]["name"]
|
||||
print(f"\n================ Node: {name} ================")
|
||||
disks = node.get("spec", {}).get("disks", {})
|
||||
status_disks = node.get("status", {}).get("diskStatus", {})
|
||||
for dname, dspec in disks.items():
|
||||
dstat = status_disks.get(dname, {})
|
||||
print(f"Disk {dname}:")
|
||||
print(f" Path: {dspec.get('path')}, AllowScheduling: {dspec.get('allowScheduling')}")
|
||||
print(f" Conditions: {dstat.get('conditions')}")
|
||||
print(f" Schedulable: {dstat.get('schedulable')}, StorageAvailable: {dstat.get('storageAvailable')}, StorageMax: {dstat.get('storageMaximum')}")
|
||||
|
||||
c.close()
|
||||
|
||||
27
scripts/check_lh_now.py
Normal file
27
scripts/check_lh_now.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Longhorn Volumes:")
|
||||
out, _ = k8s("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness,REPLICAS:.status.currentReplicas,NODE:.status.currentNodeID")
|
||||
print(out)
|
||||
|
||||
print("\nAI-Stack pods:")
|
||||
out, _ = k8s("kubectl get pods -n ai-stack")
|
||||
print(out)
|
||||
|
||||
# Check why curl to longhorn failed
|
||||
out, _ = k8s("kubectl get ep -n longhorn-system longhorn-backend")
|
||||
print("\nLonghorn-backend endpoint:")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
34
scripts/check_lh_recovery.py
Normal file
34
scripts/check_lh_recovery.py
Normal file
@@ -0,0 +1,34 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Checking Longhorn nodes disk conditions:")
|
||||
out, _ = k8s("kubectl get nodes.longhorn.io -n longhorn-system -o json")
|
||||
data = json.loads(out)
|
||||
for item in data.get("items", []):
|
||||
name = item["metadata"]["name"]
|
||||
print(f"\nNode: {name}")
|
||||
for dname, dstat in item.get("status", {}).get("diskStatus", {}).items():
|
||||
conds = {c.get("type"): c.get("status") for c in dstat.get("conditions", [])}
|
||||
storage_avail = dstat.get("storageAvailable", 0) / (1024**3)
|
||||
print(f" {dname}: Ready={conds.get('Ready')}, Schedulable={conds.get('Schedulable')}, Avail={storage_avail:.1f} GB")
|
||||
|
||||
print("\nLonghorn Volumes Status:")
|
||||
out, _ = k8s("kubectl get volumes.longhorn.io -n longhorn-system -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness")
|
||||
print(out)
|
||||
|
||||
print("\nAI-stack pods:")
|
||||
out, _ = k8s("kubectl get pods -n ai-stack")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
40
scripts/check_linux_template.py
Normal file
40
scripts/check_linux_template.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import urllib.request
|
||||
import json
|
||||
import ssl
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def zabbix_api(method, params, auth=None):
|
||||
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
|
||||
if auth: payload["auth"] = auth
|
||||
req = urllib.request.Request("https://zabbix.snarfnet.net/api_jsonrpc.php",
|
||||
data=json.dumps(payload).encode('utf-8'),
|
||||
headers={"Content-Type": "application/json-rpc"})
|
||||
resp = urllib.request.urlopen(req, context=ctx, timeout=10)
|
||||
return json.loads(resp.read().decode('utf-8'))
|
||||
|
||||
auth = zabbix_api("user.login", {"username": "Admin", "password": "zabbix"}).get("result")
|
||||
|
||||
# Get templates
|
||||
templates = zabbix_api("template.get", {
|
||||
"output": ["templateid", "name"],
|
||||
"search": {"name": "Linux"},
|
||||
"selectMacros": "extend"
|
||||
}, auth=auth).get("result", [])
|
||||
|
||||
for t in templates:
|
||||
print(f"\nTemplate: {t['name']} (ID: {t['templateid']})")
|
||||
for m in t.get("macros", []):
|
||||
if "CPU" in m['macro'] or "MEM" in m['macro'] or "LOAD" in m['macro']:
|
||||
print(f" Macro: {m['macro']} = {m['value']}")
|
||||
# Get triggers in this template
|
||||
triggers = zabbix_api("trigger.get", {
|
||||
"output": ["triggerid", "description", "expression", "priority"],
|
||||
"templateids": [t['templateid']],
|
||||
"search": {"description": "utilization"}
|
||||
}, auth=auth).get("result", [])
|
||||
for tr in triggers:
|
||||
print(f" Trigger: {tr['description']} | Expr: {tr['expression']} | Priority: {tr['priority']}")
|
||||
|
||||
13
scripts/check_local_raw_yml.py
Normal file
13
scripts/check_local_raw_yml.py
Normal file
@@ -0,0 +1,13 @@
|
||||
with open(r'c:\Users\Snarf\OneDrive\Desktop\code\SnarfCode\homelab-gitops-repo\manifests\aerodrome\prometheus\prometheus.yaml', 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
|
||||
start_marker = "prometheus.yml: |"
|
||||
start_idx = text.find(start_marker)
|
||||
end_idx = text.find("\n---", start_idx)
|
||||
raw_yml = text[start_idx + len(start_marker):end_idx]
|
||||
|
||||
print("Contains shipyard:", "shipyard" in raw_yml)
|
||||
for line in raw_yml.splitlines():
|
||||
if "shipyard" in line or "spaceport" in line or "node-exporter" in line:
|
||||
print(line)
|
||||
|
||||
22
scripts/check_mount_history.py
Normal file
22
scripts/check_mount_history.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.81", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def run_cmd(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Search for extra-disks on Node 1:")
|
||||
out, _ = run_cmd("grep -rn 'extra-disks' /oem/ /etc/ /usr/local/ 2>/dev/null")
|
||||
print(out)
|
||||
|
||||
print("\nHistory on Node 1:")
|
||||
out, _ = run_cmd("cat /home/rancher/.bash_history 2>/dev/null | tail -n 40")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_mount_unit.py
Normal file
22
scripts/check_mount_unit.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.81", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def run_cmd(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Content of var-lib-harvester-extrax2ddisks-sdb.mount:")
|
||||
out, _ = run_cmd("cat $(readlink -f /etc/systemd/system/multi-user.target.wants/var-lib-harvester-extrax2ddisks-sdb.mount 2>/dev/null || echo /etc/systemd/system/multi-user.target.wants/var-lib-harvester-extrax2ddisks-sdb.mount)")
|
||||
print(out)
|
||||
|
||||
print("\nStatus of var-lib-harvester-extrax2ddisks-sdb.mount:")
|
||||
out, err = run_cmd("systemctl status var-lib-harvester-extrax2ddisks-sdb.mount")
|
||||
print(out, err)
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_ndm.py
Normal file
22
scripts/check_ndm.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Disk manager pods:")
|
||||
out, _ = k8s("kubectl get pods -A | grep -i disk")
|
||||
print(out)
|
||||
|
||||
print("\nSample blockdevice YAML:")
|
||||
out, _ = k8s("kubectl get blockdevices.harvesterhci.io -n longhorn-system 18c78622-16ec-4be2-a833-21ad879cb7b9 -o yaml")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
18
scripts/check_ndm_logs.py
Normal file
18
scripts/check_ndm_logs.py
Normal file
@@ -0,0 +1,18 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.90", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def k8s(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml; PATH=$PATH:/var/lib/rancher/rke2/bin; {cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Harvester node-disk-manager logs:")
|
||||
out, _ = k8s("kubectl logs -n harvester-system -l app.kubernetes.io/name=harvester-node-disk-manager --tail=50")
|
||||
print(out[:2000])
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_nfs_peanut.py
Normal file
22
scripts/check_nfs_peanut.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- ls -la /mnt/nfs_shares/peanut ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S ls -la /mnt/nfs_shares/peanut")
|
||||
print(out)
|
||||
|
||||
print("--- contents of files in /mnt/nfs_shares/peanut ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S cat /mnt/nfs_shares/peanut/* 2>/dev/null")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
26
scripts/check_node5.py
Normal file
26
scripts/check_node5.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
# Note task is now on snshpd4lnxswm05 (140.44.4.75)
|
||||
c.connect('140.44.4.75', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
cid, _ = run("echo 'Katzir476!' | sudo -S docker ps -q -f name=peanut_peanut")
|
||||
print("CID on 140.44.4.75:", cid)
|
||||
|
||||
# Check what files are in the web app to see the API routes
|
||||
out, _ = run(f"echo 'Katzir476!' | sudo -S docker exec {cid} node -e 'console.log(require(\"fs\").readdirSync(\"./\"))'")
|
||||
print("App root files:\n", out)
|
||||
|
||||
# Test connecting to nutserver from 140.44.4.75 as well to be 100% sure node 05 can also reach it
|
||||
out, err = run("ping -c 2 -W 2 14.4.4.44")
|
||||
print("Ping from 140.44.4.75 to 14.4.4.44:\n", out)
|
||||
|
||||
c.close()
|
||||
|
||||
21
scripts/check_npm_conf4.py
Normal file
21
scripts/check_npm_conf4.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
def run_npm(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S docker exec $(docker ps -q -f name=nginx-proxy-manager) {cmd}"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("=== PORTAL (4.conf) ===")
|
||||
out, _ = run_npm("cat /data/nginx/proxy_host/4.conf")
|
||||
print(out)
|
||||
|
||||
print("\n=== WILDCARD (*.snarfnet.net) (1.conf) ===")
|
||||
out, _ = run_npm("cat /data/nginx/proxy_host/1.conf")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
32
scripts/check_npm_db_location.py
Normal file
32
scripts/check_npm_db_location.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
py_code = """
|
||||
import sqlite3
|
||||
import json
|
||||
|
||||
db = '/var/lib/docker/volumes/nginx-proxy-manager_data/_data/database.sqlite'
|
||||
# Check if file exists there or where npm data is
|
||||
import glob
|
||||
files = glob.glob('/var/lib/docker/**/database.sqlite', recursive=True)
|
||||
print('NPM DB files found:', files)
|
||||
if files:
|
||||
con = sqlite3.connect(files[0])
|
||||
cur = con.cursor()
|
||||
for row in cur.execute('SELECT id, domain_names, forward_host, forward_port FROM proxy_host WHERE id=4').fetchall():
|
||||
print('Host 4:', row)
|
||||
"""
|
||||
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/find_npm_db.py")
|
||||
stdin.write(py_code)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
_, o, _ = c.exec_command("echo Katzir476! | sudo -S python3 /tmp/find_npm_db.py")
|
||||
print(o.read().decode())
|
||||
|
||||
c.close()
|
||||
|
||||
@@ -1,18 +1,25 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import json
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
key_path = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="root", key_filename=key_path, timeout=5)
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
cmd = """docker service ls"""
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
print("SERVICES:")
|
||||
print(stdout.read().decode('utf-8'))
|
||||
def run_npm_sql(sql):
|
||||
cmd = f"""echo 'Katzir476!' | sudo -S docker exec -i $(echo 'Katzir476!' | sudo -S docker ps -qf name=nginx-proxy-manager_app) sqlite3 /data/database.sqlite "{sql}" """
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
# Check Proxy Host 4
|
||||
out, err = run_npm_sql("SELECT id, domain_names, forward_host, forward_port FROM proxy_host WHERE id=4;")
|
||||
print("Proxy Host 4 in NPM:")
|
||||
print(out)
|
||||
|
||||
# Check all proxy hosts with portal or heimdall
|
||||
out2, _ = run_npm_sql("SELECT id, domain_names, forward_host, forward_port FROM proxy_host WHERE domain_names LIKE '%portal%' OR domain_names LIKE '%heimdall%';")
|
||||
print("\nMatching Proxy Hosts:")
|
||||
print(out2)
|
||||
|
||||
c.close()
|
||||
|
||||
@@ -1,11 +1,29 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('140.44.4.71', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command("python3 -c \"import sqlite3; con = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite'); print(list(con.execute('SELECT id, incoming_port, forwarding_host, forwarding_port, is_deleted FROM stream')))\"")
|
||||
print("STREAMS:", stdout.read().decode())
|
||||
print("ERR:", stderr.read().decode())
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
ssh.close()
|
||||
check_streams = """
|
||||
import sqlite3
|
||||
conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
|
||||
print("NPM STREAMS:")
|
||||
for row in conn.execute("SELECT * FROM stream"):
|
||||
print(row)
|
||||
print("ALL TABLES:")
|
||||
for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'"):
|
||||
print(row)
|
||||
conn.close()
|
||||
"""
|
||||
run(f"cat << 'EOF' > /tmp/check_streams.py\n{check_streams}\nEOF")
|
||||
out, err = run("echo 'Katzir476!' | sudo -S python3 /tmp/check_streams.py")
|
||||
print(out)
|
||||
if err: print(err)
|
||||
|
||||
c.close()
|
||||
|
||||
26
scripts/check_nut_arp.py
Normal file
26
scripts/check_nut_arp.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("ip route get 140.44.4.74:")
|
||||
out, _ = run("ip route get 140.44.4.74")
|
||||
print(out)
|
||||
|
||||
print("ip route get 140.44.4.70:")
|
||||
out, _ = run("ip route get 140.44.4.70")
|
||||
print(out)
|
||||
|
||||
print("arp -n:")
|
||||
out, _ = run("arp -n")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
28
scripts/check_nut_configs.py
Normal file
28
scripts/check_nut_configs.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
KEY_FILE = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
|
||||
IP = '14.4.4.44'
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(IP, username='snarf', key_filename=KEY_FILE, timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
print(f"\n>>> {cmd}")
|
||||
_, o, e = ssh.exec_command(cmd)
|
||||
out = o.read().decode().strip()
|
||||
if out:
|
||||
print(out)
|
||||
return out
|
||||
|
||||
run("uptime")
|
||||
run("cat /etc/nut/nut.conf")
|
||||
run("cat /etc/nut/ups.conf")
|
||||
run("cat /etc/nut/upsmon.conf | grep -v '^#' | grep -v '^$'")
|
||||
run("ps aux | grep -i nut")
|
||||
run("ls -la /run/nut")
|
||||
|
||||
ssh.close()
|
||||
|
||||
33
scripts/check_nut_firewall.py
Normal file
33
scripts/check_nut_firewall.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- ping 14.4.4.20 from nutserver ---")
|
||||
out, _ = run("ping -c 2 14.4.4.20")
|
||||
print(out)
|
||||
|
||||
print("--- ping 14.44.4.108 from nutserver ---")
|
||||
out, _ = run("ping -c 2 14.44.4.108")
|
||||
print(out)
|
||||
|
||||
print("--- traceroute 140.44.4.74 from nutserver ---")
|
||||
out, _ = run("traceroute -n -m 5 140.44.4.74 || tracepath -n 140.44.4.74")
|
||||
print(out)
|
||||
|
||||
print("--- ufw / iptables on nutserver ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S ufw status verbose")
|
||||
print("UFW:", out)
|
||||
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S nft list ruleset")
|
||||
print("NFT:", out)
|
||||
|
||||
c.close()
|
||||
|
||||
26
scripts/check_nut_neigh.py
Normal file
26
scripts/check_nut_neigh.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("ip neighbor:")
|
||||
out, _ = run("ip neigh")
|
||||
print(out)
|
||||
|
||||
print("ping gateway 14.4.4.1:")
|
||||
out, _ = run("ping -c 2 14.4.4.1")
|
||||
print(out)
|
||||
|
||||
print("ip neigh after ping:")
|
||||
out, _ = run("ip neigh")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
26
scripts/check_nut_routes.py
Normal file
26
scripts/check_nut_routes.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- ping 140.44.4.1 (cluster gateway) ---")
|
||||
out, _ = run("ping -c 2 140.44.4.1")
|
||||
print(out)
|
||||
|
||||
print("--- ping 1.1.1.1 (internet) ---")
|
||||
out, _ = run("ping -c 2 1.1.1.1")
|
||||
print(out)
|
||||
|
||||
print("--- default route & ip route show table all ---")
|
||||
out, _ = run("ip route show table all")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_nut_wifi.py
Normal file
22
scripts/check_nut_wifi.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- iwconfig / nmcli on nutserver ---")
|
||||
out, _ = run("iwconfig 2>&1 || nmcli 2>&1")
|
||||
print(out)
|
||||
|
||||
print("--- SSID / Network connection ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S grep -h ssid /etc/wpa_supplicant/wpa_supplicant.conf /etc/NetworkManager/system-connections/* 2>/dev/null")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
43
scripts/check_nutserver.py
Normal file
43
scripts/check_nutserver.py
Normal file
@@ -0,0 +1,43 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
print("Connected to 14.4.4.44 via key")
|
||||
except Exception as e:
|
||||
print("Key auth failed, trying password:", e)
|
||||
c.connect('14.4.4.44', username='snarf', password='Katzir476!', timeout=10)
|
||||
print("Connected to 14.4.4.44 via password")
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- Listening ports on nutserver ---")
|
||||
out, _ = run("ss -tulpn | grep 3493 || netstat -tulpn | grep 3493")
|
||||
print(out)
|
||||
|
||||
print("\n--- /etc/nut/upsd.conf ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S cat /etc/nut/upsd.conf | grep -v '^#' | grep -v '^$'")
|
||||
print(out)
|
||||
|
||||
print("\n--- /etc/nut/nut.conf ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S cat /etc/nut/nut.conf | grep -v '^#' | grep -v '^$'")
|
||||
print(out)
|
||||
|
||||
print("\n--- /etc/nut/upsd.users ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S cat /etc/nut/upsd.users | grep -v '^#' | grep -v '^$'")
|
||||
print(out)
|
||||
|
||||
print("\n--- /etc/nut/ups.conf ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S cat /etc/nut/ups.conf | grep -v '^#' | grep -v '^$'")
|
||||
print(out)
|
||||
|
||||
print("\n--- upsc server-closet@localhost ---")
|
||||
out, _ = run("upsc server-closet@localhost")
|
||||
print(out[:300])
|
||||
|
||||
c.close()
|
||||
@@ -2,19 +2,32 @@ import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
key_path = r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet'
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("14.4.4.44", username="root", key_filename=key_path, timeout=10)
|
||||
c.connect('14.4.4.44', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode('utf-8', errors='ignore').strip()
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("iptables check:", run("iptables --version; which iptables nft ipset"))
|
||||
print("ip forwarding:", run("cat /proc/sys/net/ipv4/ip_forward"))
|
||||
print("canary test ping to gateway:", run("ping -c 2 14.4.4.1"))
|
||||
print("--- nutserver ip a ---")
|
||||
out, _ = run("ip a")
|
||||
print(out)
|
||||
|
||||
print("--- nutserver ip route ---")
|
||||
out, _ = run("ip route")
|
||||
print(out)
|
||||
|
||||
print("--- nutserver ping gateway ---")
|
||||
out, _ = run("ping -c 2 14.4.4.1 || ping -c 2 14.4.4.254")
|
||||
print(out)
|
||||
|
||||
print("--- nutserver iptables / nftables ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S iptables -L -n -v")
|
||||
print(out)
|
||||
|
||||
print("--- nutserver ping 140.44.4.74 ---")
|
||||
out, _ = run("ping -c 2 140.44.4.74")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
|
||||
26
scripts/check_one_mount.py
Normal file
26
scripts/check_one_mount.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import paramiko
|
||||
|
||||
SSH_KEY = r"C:\Users\Snarf\.ssh\id_ed25519_snarfnet"
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.81", username="rancher", key_filename=SSH_KEY, timeout=10)
|
||||
|
||||
def run_cmd(cmd):
|
||||
full_cmd = f"echo Katzir476! | sudo -S sh -c '{cmd}'"
|
||||
_, o, e = c.exec_command(full_cmd)
|
||||
return o.read().decode().strip(), e.read().decode().strip()
|
||||
|
||||
print("Files in /etc/systemd/system matching var-lib*:")
|
||||
out, _ = run_cmd("ls -la /etc/systemd/system/var-lib*")
|
||||
print(out)
|
||||
|
||||
print("\nTry running systemctl start var-lib-harvester-extrax2ddisks-sdb.mount:")
|
||||
out, err = run_cmd("systemctl start var-lib-harvester-extrax2ddisks-sdb.mount")
|
||||
print("out:", out, "err:", err)
|
||||
|
||||
print("\nStatus:")
|
||||
out, _ = run_cmd("systemctl status var-lib-harvester-extrax2ddisks-sdb.mount")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
24
scripts/check_panel_targets.py
Normal file
24
scripts/check_panel_targets.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
resp = requests.get("https://grafana.snarfnet.net/api/dashboards/uid/snarfnet-systems", auth=auth, verify=False)
|
||||
dash = resp.json()["dashboard"]
|
||||
print("--- Systems Dashboard Targets ---")
|
||||
for p in dash.get("panels", []):
|
||||
targets = p.get("targets", [])
|
||||
if targets:
|
||||
exprs = [t.get("expr") for t in targets]
|
||||
print(f"Panel: {p.get('title')} -> Exprs: {exprs}")
|
||||
|
||||
resp2 = requests.get("https://grafana.snarfnet.net/api/dashboards/uid/snarfnet-containers", auth=auth, verify=False)
|
||||
dash2 = resp2.json()["dashboard"]
|
||||
print("\n--- Containers Dashboard Targets ---")
|
||||
for p in dash2.get("panels", []):
|
||||
targets = p.get("targets", [])
|
||||
if targets:
|
||||
exprs = [t.get("expr") for t in targets]
|
||||
print(f"Panel: {p.get('title')} -> Exprs: {exprs}")
|
||||
|
||||
45
scripts/check_peanut_access.py
Normal file
45
scripts/check_peanut_access.py
Normal file
@@ -0,0 +1,45 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, o, e = c.exec_command(cmd)
|
||||
return o.read().decode().strip()
|
||||
|
||||
# Check direct curl to port 8050 on Swarm VIP and nodes
|
||||
for ip in ["140.44.4.70", "140.44.4.71", "140.44.4.74"]:
|
||||
res = run(f"curl -s -o /dev/null -w '%{{http_code}}' http://{ip}:8050")
|
||||
print(f"Curl http://{ip}:8050 -> HTTP {res}")
|
||||
|
||||
# Check NPM database
|
||||
script = '''
|
||||
import sqlite3
|
||||
conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
|
||||
cur = conn.cursor()
|
||||
rows = cur.execute("SELECT id, domain_names, forward_host, forward_port, enabled FROM proxy_host").fetchall()
|
||||
for r in rows:
|
||||
if any(k in str(r).lower() for k in ["peanut", "ups", "8050"]):
|
||||
print(r)
|
||||
conn.close()
|
||||
'''
|
||||
out = run(f"echo 'Katzir476!' | sudo -S python3 -c {repr(script)}")
|
||||
print("\nNPM matching records:\n", out)
|
||||
|
||||
# Check Heimdall tile for PeaNUT
|
||||
heimdall_script = '''
|
||||
import sqlite3
|
||||
con = sqlite3.connect('/var/lib/docker-data/heimdall/config/www/app.sqlite')
|
||||
cur = con.cursor()
|
||||
row = cur.execute("SELECT id, title, url FROM items WHERE title LIKE '%peanut%' OR title LIKE '%nut%'").fetchall()
|
||||
print("Heimdall tile for PeaNUT:", row)
|
||||
con.close()
|
||||
'''
|
||||
out_h = run(f"echo 'Katzir476!' | sudo -S python3 -c {repr(heimdall_script)}")
|
||||
print(out_h)
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_peanut_device.py
Normal file
22
scripts/check_peanut_device.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- PeaNUT Container Logs ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S docker service logs peanut_peanut --tail 30")
|
||||
print(out)
|
||||
|
||||
print("\n--- Curl http://140.44.4.70:8050/device/server-closet ---")
|
||||
out, _ = run("curl -s http://140.44.4.70:8050/device/server-closet | grep -i -E 'battery|charge|runtime|cyberpower|or2200|server-closet' | head -n 30")
|
||||
print("Matches in HTML:\n", out[:500])
|
||||
|
||||
c.close()
|
||||
|
||||
22
scripts/check_peanut_history.py
Normal file
22
scripts/check_peanut_history.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- History of /mnt/swarm_shared/legacy_volumes/peanut ---")
|
||||
out, _ = run("ls -la /mnt/swarm_shared/legacy_volumes/peanut")
|
||||
print(out)
|
||||
|
||||
print("--- Any other peanut directories on NFS / storage ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S find /mnt -iname '*peanut*' 2>/dev/null")
|
||||
print(out)
|
||||
|
||||
c.close()
|
||||
|
||||
32
scripts/check_peanut_logs.py
Normal file
32
scripts/check_peanut_logs.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.74', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
stdin, stdout, stderr = c.exec_command(cmd, timeout=10)
|
||||
return stdout.read().decode().strip(), stderr.read().decode().strip()
|
||||
|
||||
print("--- Container logs on 140.44.4.74 ---")
|
||||
cid, _ = run("echo 'Katzir476!' | sudo -S docker ps -q -f name=peanut_peanut")
|
||||
print("CID:", cid)
|
||||
if cid:
|
||||
logs, _ = run(f"echo 'Katzir476!' | sudo -S docker logs --tail 30 {cid}")
|
||||
print("LOGS:\n", logs)
|
||||
|
||||
print("\n--- Files in /mnt/swarm_shared/legacy_volumes/peanut ---")
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S ls -la /mnt/swarm_shared/legacy_volumes/peanut")
|
||||
print(out)
|
||||
|
||||
# Check what SQLite or config files are inside /mnt/swarm_shared/legacy_volumes/peanut
|
||||
out, _ = run("echo 'Katzir476!' | sudo -S find /mnt/swarm_shared/legacy_volumes/peanut -type f")
|
||||
print("Files found:\n", out)
|
||||
|
||||
# If db exists, check tables / contents
|
||||
out, _ = run("python3 -c 'import urllib.request; print(urllib.request.urlopen(\"http://140.44.4.70:8050/api/devices\").read().decode())' 2>&1")
|
||||
print("API /api/devices:", out)
|
||||
|
||||
c.close()
|
||||
33
scripts/check_peanut_status.py
Normal file
33
scripts/check_peanut_status.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
print(f"\n>>> {cmd}")
|
||||
_, o, e = c.exec_command(cmd)
|
||||
out = o.read().decode().strip()
|
||||
if out:
|
||||
print(out)
|
||||
return out
|
||||
|
||||
print("--- PeaNUT Service Status ---")
|
||||
run("echo 'Katzir476!' | sudo -S docker service ps peanut_peanut")
|
||||
run("echo 'Katzir476!' | sudo -S docker service logs peanut_peanut --tail 30")
|
||||
|
||||
print("\n--- Check PeaNUT NPM Proxy Host ---")
|
||||
npm_query = """import sqlite3
|
||||
conn = sqlite3.connect('/var/lib/docker/volumes/nginx-proxy-manager_nginx-data/_data/database.sqlite')
|
||||
cur = conn.cursor()
|
||||
rows = cur.execute("SELECT id, domain_names, forward_host, forward_port, enabled FROM proxy_host WHERE domain_names LIKE '%peanut%' OR domain_names LIKE '%ups%' OR forward_port=8050").fetchall()
|
||||
for r in rows:
|
||||
print(r)
|
||||
conn.close()
|
||||
"""
|
||||
run(f"""echo 'Katzir476!' | sudo -S python3 -c "{npm_query}" """)
|
||||
|
||||
c.close()
|
||||
|
||||
28
scripts/check_pings_from_clusters.py
Normal file
28
scripts/check_pings_from_clusters.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
hosts = [
|
||||
('140.44.4.34', 'Aerodrome 04'),
|
||||
('140.44.4.41', 'Spaceport 01'),
|
||||
('140.44.4.71', 'Shipyard 01'),
|
||||
('140.44.4.81', 'Spacedock 01'),
|
||||
]
|
||||
|
||||
for ip, name in hosts:
|
||||
try:
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
if ip == '140.44.4.41':
|
||||
c.connect(ip, username='root', password='dietpi', timeout=5)
|
||||
else:
|
||||
c.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
|
||||
_, stdout, _ = c.exec_command("ping -c 2 -W 2 14.4.4.44")
|
||||
res = stdout.read().decode().strip()
|
||||
has_reply = "64 bytes from 14.4.4.44" in res
|
||||
print(f"{name} ({ip}) -> 14.4.4.44: {'REPLIED' if has_reply else 'NO REPLY'}")
|
||||
c.close()
|
||||
except Exception as e:
|
||||
print(f"{name} ({ip}) error: {e}")
|
||||
|
||||
9
scripts/check_pod_shipyard.py
Normal file
9
scripts/check_pod_shipyard.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl exec deploy/prometheus -- cat /etc/prometheus/prometheus.yml | grep -i shipyard")
|
||||
print("Shipyard in pod /etc/prometheus/prometheus.yml:\n", o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
@@ -1,24 +1,8 @@
|
||||
import socket
|
||||
|
||||
hosts = [
|
||||
('140.44.4.11', 9100, 'Aerodrome Master 01'),
|
||||
('140.44.4.41', 9100, 'Spaceport Master 01'),
|
||||
('140.44.4.71', 9100, 'Shipyard Swarm'),
|
||||
('140.44.4.81', 9100, 'Harvester Node 01'),
|
||||
('140.44.4.81', 9400, 'Harvester Node 01 DCGM'),
|
||||
('140.44.4.11', 10250, 'Aerodrome Kubelet'),
|
||||
('140.44.4.41', 10250, 'Spaceport Kubelet'),
|
||||
('140.44.4.60', 8888, 'Zabbix Web'),
|
||||
('140.44.4.64', 10051, 'Zabbix Server'),
|
||||
]
|
||||
|
||||
for ip, port, desc in hosts:
|
||||
s = socket.socket()
|
||||
s.settimeout(1.5)
|
||||
try:
|
||||
s.connect((ip, port))
|
||||
print(f"[OPEN] {ip}:{port} ({desc})")
|
||||
s.close()
|
||||
except Exception as e:
|
||||
print(f"[CLOSED/ERR] {ip}:{port} ({desc}) - {e}")
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("echo 'Katzir476!' | sudo -S ss -tulpn | grep -E ':9100|:9101|:8080'")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
15
scripts/check_ports_all.py
Normal file
15
scripts/check_ports_all.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import paramiko
|
||||
|
||||
nodes = ['140.44.4.71', '140.44.4.72', '140.44.4.73', '140.44.4.74', '140.44.4.75']
|
||||
for ip in nodes:
|
||||
try:
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(ip, username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=5)
|
||||
_, o, _ = c.exec_command("echo 'Katzir476!' | sudo -S ss -tulpn | grep -E ':9100|:9101|:8080'")
|
||||
out = o.read().decode().strip()
|
||||
print(f"Node {ip}: {out if out else 'All clear'}")
|
||||
c.close()
|
||||
except Exception as e:
|
||||
print(f"Node {ip} err: {e}")
|
||||
|
||||
15
scripts/check_prom_all_targets.py
Normal file
15
scripts/check_prom_all_targets.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
def prom_query(q):
|
||||
url = "https://grafana.snarfnet.net/api/datasources/proxy/1/api/v1/query"
|
||||
resp = requests.get(url, params={"query": q}, auth=auth, verify=False, timeout=10)
|
||||
return resp.json().get("data", {}).get("result", [])
|
||||
|
||||
res = prom_query("count by (instance, job, cluster) (up)")
|
||||
for r in res:
|
||||
print(r.get("metric"))
|
||||
|
||||
16
scripts/check_prom_cadvisor_nodes.py
Normal file
16
scripts/check_prom_cadvisor_nodes.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
def prom_query(q):
|
||||
url = "https://grafana.snarfnet.net/api/datasources/proxy/1/api/v1/query"
|
||||
resp = requests.get(url, params={"query": q}, auth=auth, verify=False, timeout=10)
|
||||
return resp.json().get("data", {}).get("result", [])
|
||||
|
||||
res = prom_query("count by (instance, node) (container_last_seen{container!=''})")
|
||||
print("Container nodes in Prometheus:")
|
||||
for r in res:
|
||||
print(r.get("metric"))
|
||||
|
||||
8
scripts/check_prom_cm.py
Normal file
8
scripts/check_prom_cm.py
Normal file
@@ -0,0 +1,8 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get configmap prometheus-config -o yaml | grep -A 40 'job_name: node-exporter'")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
9
scripts/check_prom_content.py
Normal file
9
scripts/check_prom_content.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl exec deploy/prometheus -- grep -C 5 spaceport /etc/prometheus/prometheus.yml")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
11
scripts/check_prom_in_pod.py
Normal file
11
scripts/check_prom_in_pod.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl exec deploy/prometheus -- cat /etc/prometheus/prometheus.yml")
|
||||
content = o.read().decode().strip()
|
||||
lines = content.splitlines()
|
||||
for l in lines[76:125]:
|
||||
print(l)
|
||||
c.close()
|
||||
28
scripts/check_prom_labels.py
Normal file
28
scripts/check_prom_labels.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
auth = ("admin", "Katzir476!!!")
|
||||
def prom_query(q):
|
||||
url = "https://grafana.snarfnet.net/api/datasources/proxy/1/api/v1/query"
|
||||
resp = requests.get(url, params={"query": q}, auth=auth, verify=False, timeout=10)
|
||||
return resp.json().get("data", {}).get("result", [])
|
||||
|
||||
res_nodes = prom_query("node_load1")
|
||||
print(f"Node Exporter targets ({len(res_nodes)}):")
|
||||
for r in res_nodes:
|
||||
m = r.get("metric", {})
|
||||
print(f" instance: {m.get('instance')}, job: {m.get('job')}, nodename: {m.get('nodename')}, cluster: {m.get('cluster')}")
|
||||
|
||||
res_containers = prom_query("count by (cluster, job, node) (container_last_seen{container!=''})")
|
||||
print("\nContainer metric groupings:")
|
||||
for r in res_containers:
|
||||
print(" ", r.get("metric"))
|
||||
|
||||
# Check all label values for 'cluster' or 'job'
|
||||
res_jobs = prom_query("count by (job) (up)")
|
||||
print("\nPrometheus jobs:")
|
||||
for r in res_jobs:
|
||||
print(" ", r.get("metric"))
|
||||
|
||||
13
scripts/check_prom_mounts.py
Normal file
13
scripts/check_prom_mounts.py
Normal file
@@ -0,0 +1,13 @@
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get deploy prometheus -o json")
|
||||
data = json.loads(o.read().decode().strip())
|
||||
c_spec = data.get("spec", {}).get("template", {}).get("spec", {}).get("containers", [])[0]
|
||||
print("Args:", c_spec.get("args"))
|
||||
print("VolumeMounts:", c_spec.get("volumeMounts"))
|
||||
c.close()
|
||||
|
||||
9
scripts/check_prom_ns.py
Normal file
9
scripts/check_prom_ns.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('140.44.4.11', username='root', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
|
||||
_, o, _ = c.exec_command("kubectl get configmap -A | grep prometheus")
|
||||
print(o.read().decode().strip())
|
||||
c.close()
|
||||
|
||||
46
scripts/check_real_heimdall_db.py
Normal file
46
scripts/check_real_heimdall_db.py
Normal file
@@ -0,0 +1,46 @@
|
||||
import paramiko
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("140.44.4.71", username="snarf", password="Katzir476!", timeout=10)
|
||||
|
||||
py_code = '''
|
||||
import sqlite3
|
||||
import shutil
|
||||
import time
|
||||
|
||||
db_path = '/var/lib/docker-data/heimdall/config/www/app.sqlite'
|
||||
backup_name = db_path + '.backup_live_' + str(int(time.time()))
|
||||
shutil.copyfile(db_path, backup_name)
|
||||
print('Backup created:', backup_name)
|
||||
|
||||
con = sqlite3.connect(db_path)
|
||||
cur = con.cursor()
|
||||
|
||||
print('=== USERS ===')
|
||||
for r in cur.execute('SELECT * FROM users').fetchall():
|
||||
print('User:', r)
|
||||
|
||||
print('\\n=== ITEMS ===')
|
||||
rows = cur.execute('SELECT id, title, url, pinned, user_id, apporder FROM items').fetchall()
|
||||
print('Total items in DB:', len(rows))
|
||||
for r in rows:
|
||||
print('Item:', r)
|
||||
|
||||
print('\\n=== SETTINGS ===')
|
||||
for r in cur.execute('SELECT * FROM settings').fetchall():
|
||||
print('Setting:', r)
|
||||
'''
|
||||
|
||||
stdin, stdout, stderr = c.exec_command("cat > /tmp/inspect_heimdall_real.py")
|
||||
stdin.write(py_code)
|
||||
stdin.close()
|
||||
stdout.channel.recv_exit_status()
|
||||
|
||||
_, o, e = c.exec_command("echo Katzir476! | sudo -S python3 /tmp/inspect_heimdall_real.py")
|
||||
print(o.read().decode())
|
||||
if e.read().decode():
|
||||
print("ERR:", e.read().decode())
|
||||
|
||||
c.close()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user