feat: BSN Watchdog — überwacht Flask, Tunnel, Article Server, DB, RAM alle 5 Min

2026-06-20 09:03:49 +02:00
parent c3b0e8fcf2
commit 1dc63a4eda
1 changed files with 256 additions and 0 deletions
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+"""BSN Watchdog — Überwacht Flask + Cloudflare Tunnel + Article Server.
+   Läuft als Cron-Job alle 5 Minuten.
+   Sendet Alert per Telegram nur bei Zustands-ÄNDERUNG (nicht bei jedem Check).
+"""
+
+import subprocess
+import sys
+import json
+import os
+from datetime import datetime
+
+NOW = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+STATE_FILE = "/tmp/bsn_watchdog_state.json"
+ALERT_COOLDOWN = 900  # 15 Minuten zwischen Alerts
+
+# ── Telegram (optional) ──
+TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN", "")
+TELEGRAM_CHAT_ID = os.environ.get("ADMIN_TELEGRAM_CHAT_ID", "")
+if not TELEGRAM_CHAT_ID:
+    # Fallback: aus Hermes Config lesen
+    try:
+        import yaml
+        with open(os.path.expanduser("~/.hermes/config.yaml")) as f:
+            cfg = yaml.safe_load(f)
+        home_channel = cfg.get("messaging", {}).get("telegram", {}).get("home_channel", "")
+        if home_channel:
+            TELEGRAM_CHAT_ID = home_channel
+    except:
+        pass
+
+def send_alert(msg: str):
+    """Sende Telegram-Alert (nur wenn Token + Chat ID vorhanden)."""
+    if not TELEGRAM_TOKEN or not TELEGRAM_CHAT_ID:
+        print(f"[alert] Telegram nicht konfiguriert: {msg}")
+        return
+    try:
+        url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage"
+        subprocess.run([
+            "curl", "-s", "-X", "POST", url,
+            "-H", "Content-Type: application/json",
+            "-d", json.dumps({
+                "chat_id": TELEGRAM_CHAT_ID,
+                "text": msg,
+                "parse_mode": "Markdown"
+            })
+        ], timeout=10, capture_output=True)
+    except Exception as e:
+        print(f"[alert] Fehler: {e}")
+
+def load_state():
+    try:
+        with open(STATE_FILE) as f:
+            return json.load(f)
+    except:
+        return {"last_alert": 0, "last_status": {}}
+
+def save_state(state):
+    with open(STATE_FILE, "w") as f:
+        json.dump(state, f)
+
+def check_port(port: int) -> bool:
+    """Prüft ob localhost:PORT antwortet."""
+    try:
+        r = subprocess.run(
+            ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
+             f"http://localhost:{port}/", "--max-time", "5"],
+            capture_output=True, text=True, timeout=6
+        )
+        return r.stdout.strip() == "200"
+    except:
+        return False
+
+def check_url(url: str) -> bool:
+    """Prüft ob externe URL antwortet."""
+    try:
+        r = subprocess.run(
+            ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
+             url, "--max-time", "10"],
+            capture_output=True, text=True, timeout=12
+        )
+        return r.stdout.strip() in ("200", "302", "301")
+    except:
+        return False
+
+def check_process(name_pattern: str) -> bool:
+    """Prüft ob Prozess läuft."""
+    try:
+        r = subprocess.run(
+            ["pgrep", "-f", name_pattern],
+            capture_output=True, timeout=3
+        )
+        return r.returncode == 0
+    except:
+        return False
+
+def try_recover(service: str) -> str:
+    """Versucht einen Service wiederherzustellen."""
+    if service == "flask":
+        subprocess.Popen(
+            ["/home/hermes/venv/bin/python", "app.py"],
+            cwd="/home/hermes/workspace/bsn-chatbot",
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
+        return "Flask neugestartet"
+    elif service == "tunnel":
+        subprocess.Popen(
+            ["cloudflared", "tunnel", "--config",
+             "/home/hermes/.cloudflared/config.yml", "run"],
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
+        return "Tunnel neugestartet"
+    elif service == "article_server":
+        subprocess.Popen(
+            ["/home/hermes/venv/bin/python", "article_server.py"],
+            cwd="/home/hermes/workspace",
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
+        return "Article-Server neugestartet"
+    return ""
+
+# ── Checks ──
+results = {}
+
+# 1. Flask
+flask_process = check_process("python.*app.py")
+flask_port = check_port(5002)
+results["flask"] = {
+    "process": flask_process,
+    "port_5002": flask_port,
+    "status": "ok" if (flask_process and flask_port) else "down"
+}
+
+# 2. Tunnel
+tunnel_process = check_process("cloudflared.*tunnel")
+tunnel_url = check_url("https://chat.datenhimmel.work/health")
+results["tunnel"] = {
+    "process": tunnel_process,
+    "url": tunnel_url,
+    "status": "ok" if (tunnel_process and tunnel_url) else "down"
+}
+
+# 3. Article Server (optional)
+article_process = check_process("python.*article_server")
+article_port = check_port(8890)
+results["article_server"] = {
+    "process": article_process,
+    "port_8890": article_port,
+    "status": "ok" if (article_process and article_port) else "down"
+}
+
+# 4. SQLite DB
+try:
+    db_size = os.path.getsize("/home/hermes/workspace/bsn-chatbot/bsn_intake.db")
+    results["database"] = {"status": "ok", "size_mb": round(db_size/1024/1024, 1)}
+except:
+    results["database"] = {"status": "down"}
+
+# 5. Speicher
+try:
+    r = subprocess.run(["free", "-m"], capture_output=True, text=True, timeout=3)
+    mem_line = [l for l in r.stdout.split("\n") if "Mem:" in l]
+    if mem_line:
+        parts = mem_line[0].split()
+        results["memory"] = {
+            "total_mb": int(parts[1]),
+            "used_mb": int(parts[2]),
+            "available_mb": int(parts[6]),
+            "status": "ok" if int(parts[6]) > 500 else "warn"
+        }
+except:
+    results["memory"] = {"status": "unknown"}
+
+# ── Status-Zusammenfassung ──
+all_ok = all(r.get("status") == "ok" for r in results.values())
+down_services = [name for name, r in results.items() if r.get("status") == "down"]
+warn_services = [name for name, r in results.items() if r.get("status") == "warn"]
+
+# ── State-Management ──
+state = load_state()
+prev_status = state.get("last_status", {})
+now_ts = int(datetime.now().timestamp())
+
+# Status-Änderungen erkennen
+changed = {}
+for name, r in results.items():
+    old = prev_status.get(name, "unknown")
+    new = r.get("status", "unknown")
+    if old != new:
+        changed[name] = f"{old} → {new}"
+
+# ── Ausgabe ──
+status_line = "🟢" if all_ok else "🔴" if down_services else "🟡"
+print(f"{status_line} BSN Watchdog {NOW}")
+print(f"   Flask:      {'✅' if results['flask']['status']=='ok' else '❌'} (PID={'✓' if flask_process else '✗'} Port={'✓' if flask_port else '✗'})")
+print(f"   Tunnel:     {'✅' if results['tunnel']['status']=='ok' else '❌'} (PID={'✓' if tunnel_process else '✗'} URL={'✓' if tunnel_url else '✗'})")
+print(f"   Article:    {'✅' if results['article_server']['status']=='ok' else '⚪' if article_process else '—'} (PID={'✓' if article_process else '✗'} Port={'✓' if article_port else '✗'})")
+print(f"   DB:         {'✅' if results['database']['status']=='ok' else '❌'} ({results['database'].get('size_mb','?')} MB)")
+if "memory" in results:
+    mem = results["memory"]
+    print(f"   RAM:        {mem.get('available_mb','?')} MB frei / {mem.get('total_mb','?')} MB")
+
+# ── Alerts nur bei Änderung oder nach Cooldown ──
+if down_services:
+    should_alert = False
+    if changed:
+        should_alert = True  # Neuer Ausfall
+    elif now_ts - state.get("last_alert", 0) > ALERT_COOLDOWN:
+        should_alert = True  # Periodische Erinnerung
+
+    if should_alert:
+        alert_parts = [f"🔴 *BSN Watchdog — {len(down_services)} Service(s) down*\n"]
+        for svc in down_services:
+            r = results[svc]
+            details = []
+            if "process" in r: details.append(f"Prozess={'✓' if r['process'] else '✗'}")
+            if "port_5002" in r: details.append(f"Port={'✓' if r['port_5002'] else '✗'}")
+            if "port_8890" in r: details.append(f"Port={'✓' if r['port_8890'] else '✗'}")
+            if "url" in r: details.append(f"URL={'✓' if r['url'] else '✗'}")
+            alert_parts.append(f"• *{svc}*: {', '.join(details)}")
+
+            # Auto-Recovery
+            if svc in ("flask", "tunnel", "article_server"):
+                msg = try_recover(svc)
+                alert_parts.append(f"  ↳ {msg}")
+
+        if changed:
+            alert_parts.append(f"\n_Geändert: {', '.join(f'{k}: {v}' for k,v in changed.items())}_")
+        alert_parts.append(f"\n_{NOW}_")
+        
+        send_alert("\n".join(alert_parts))
+        state["last_alert"] = now_ts
+
+elif all_ok and prev_status and any(
+    prev_status.get(n, "ok") != "ok" for n in results
+):
+    # Recovery-Alert
+    recovered = [n for n in results if prev_status.get(n, "ok") != "ok"]
+    msg = f"🟢 *BSN Watchdog — Alle Services wieder online!*\nErholt: {', '.join(recovered)}\n_{NOW}_"
+    send_alert(msg)
+    state["last_alert"] = now_ts
+
+# ── State speichern ──
+state["last_status"] = {name: r["status"] for name, r in results.items()}
+save_state(state)
+
+# ── Exit-Code ──
+if down_services:
+    print(f"\n❌ DOWN: {', '.join(down_services)}")
+    sys.exit(1)
+elif warn_services:
+    print(f"\n⚠️ WARN: {', '.join(warn_services)}")
+    sys.exit(0)
+else:
+    print("\n✅ Alle Systeme normal")
+    sys.exit(0)