feat: add infrastructure automation scripts and master improvement plan

scripts/deploy_swarm_backup_timer.py

@@ -0,0 +1,144 @@
+import paramiko
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+
+backup_script = """#!/usr/bin/env python3
+import sqlite3
+import os
+import sys
+import tarfile
+import datetime
+import glob
+
+TIMESTAMP = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+RETENTION_DAYS = 30
+
+SERVICES = [
+    {
+        "name": "gotify",
+        "src_db": "/mnt/swarm_shared/legacy_volumes/gotify/_data/gotify.db",
+        "backup_dir": "/mnt/nfs_shares/gotify/backups",
+        "extra_files": [
+            "/mnt/swarm_shared/legacy_volumes/gotify/_data/discord_bridge.py",
+            "/mnt/swarm_shared/legacy_volumes/gotify/_data/discord_bridge_last_id.txt",
+            "/mnt/swarm_shared/legacy_volumes/gotify/_data/ha_last_logbook_when.txt"
+        ]
+    },
+    {
+        "name": "homebox",
+        "src_db": "/mnt/swarm_shared/legacy_volumes/homebox/homebox.db",
+        "backup_dir": "/mnt/nfs_shares/homebox/backups",
+        "extra_files": []
+    }
+]
+
+print(f"[{datetime.datetime.now()}] === Starting Docker Swarm Automated SQLite Backups ===")
+
+for svc in SERVICES:
+    name = svc["name"]
+    src_db = svc["src_db"]
+    backup_dir = svc["backup_dir"]
+    archive_dir = os.path.join(backup_dir, "archives")
+    os.makedirs(archive_dir, exist_ok=True)
+    
+    if not os.path.exists(src_db):
+        print(f"[{datetime.datetime.now()}] [WARN] Source database not found for {name}: {src_db}")
+        continue
+
+    staging_db = os.path.join(backup_dir, f"{name}_atomic_{TIMESTAMP}.db")
+    archive_tar = os.path.join(archive_dir, f"{name}_backup_{TIMESTAMP}.tar.gz")
+
+    try:
+        print(f"[{datetime.datetime.now()}] [{name}] Executing atomic VACUUM INTO...")
+        con = sqlite3.connect(src_db)
+        con.execute(f"VACUUM INTO '{staging_db}'")
+        con.close()
+        db_size = os.path.getsize(staging_db)
+        print(f"[{datetime.datetime.now()}] [{name}] Atomic snapshot created: {db_size} bytes")
+
+        print(f"[{datetime.datetime.now()}] [{name}] Creating compressed archive: {archive_tar}...")
+        with tarfile.open(archive_tar, "w:gz") as tar:
+            tar.add(staging_db, arcname=f"{name}.db")
+            for extra in svc.get("extra_files", []):
+                if os.path.exists(extra):
+                    tar.add(extra, arcname=os.path.basename(extra))
+
+        os.remove(staging_db)
+        tar_size = os.path.getsize(archive_tar)
+        print(f"[{datetime.datetime.now()}] [{name}] Archive completed: {tar_size} bytes")
+
+        # Retention pruning
+        now = datetime.datetime.now()
+        for f in glob.glob(os.path.join(archive_dir, f"{name}_backup_*.tar.gz")):
+            mtime = datetime.datetime.fromtimestamp(os.path.getmtime(f))
+            if (now - mtime).days > RETENTION_DAYS:
+                print(f"[{datetime.datetime.now()}] [{name}] Pruning old archive: {f}")
+                os.remove(f)
+
+    except Exception as e:
+        print(f"[{datetime.datetime.now()}] [ERROR] Backup failed for {name}: {e}", file=sys.stderr)
+        if os.path.exists(staging_db):
+            os.remove(staging_db)
+
+print(f"[{datetime.datetime.now()}] === Docker Swarm SQLite Backups Finished Successfully ===")
+"""
+
+service_unit = """[Unit]
+Description=Docker Swarm SQLite Online Automated Backup Service
+After=network-online.target remote-fs.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/swarm_sqlite_backup.py
+User=root
+StandardOutput=journal
+StandardError=journal
+"""
+
+timer_unit = """[Unit]
+Description=Trigger Docker Swarm SQLite Automated Backup Daily
+
+[Timer]
+OnCalendar=*-*-* 03:30:00
+Persistent=true
+RandomizedDelaySec=300
+
+[Install]
+WantedBy=timers.target
+"""
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('140.44.4.71', username='snarf', key_filename=r'C:\Users\Snarf\.ssh\id_ed25519_snarfnet', timeout=10)
+
+sftp = ssh.open_sftp()
+with sftp.file('/tmp/swarm_sqlite_backup.py', 'w') as f:
+    f.write(backup_script)
+with sftp.file('/tmp/swarm-sqlite-backup.service', 'w') as f:
+    f.write(service_unit)
+with sftp.file('/tmp/swarm-sqlite-backup.timer', 'w') as f:
+    f.write(timer_unit)
+sftp.close()
+
+cmds = [
+    'sudo mv /tmp/swarm_sqlite_backup.py /usr/local/bin/swarm_sqlite_backup.py',
+    'sudo chmod 755 /usr/local/bin/swarm_sqlite_backup.py',
+    'sudo mv /tmp/swarm-sqlite-backup.service /etc/systemd/system/swarm-sqlite-backup.service',
+    'sudo mv /tmp/swarm-sqlite-backup.timer /etc/systemd/system/swarm-sqlite-backup.timer',
+    'sudo systemctl daemon-reload',
+    'sudo systemctl enable --now swarm-sqlite-backup.timer',
+    'sudo systemctl start swarm-sqlite-backup.service',
+    'sudo systemctl status swarm-sqlite-backup.timer --no-pager',
+    'sudo journalctl -u swarm-sqlite-backup.service -n 25 --no-pager'
+]
+
+for cmd in cmds:
+    print(f">>> {cmd}")
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    print(stdout.read().decode())
+    err = stderr.read().decode()
+    if err:
+        print("STDERR:", err)
+
+ssh.close()