diff --git a/scripts/watchdog.py b/scripts/watchdog.py new file mode 100644 index 0000000..21aa099 --- /dev/null +++ b/scripts/watchdog.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""BSN Watchdog — Überwacht Flask + Cloudflare Tunnel + Article Server. + Läuft als Cron-Job alle 5 Minuten. + Sendet Alert per Telegram nur bei Zustands-ÄNDERUNG (nicht bei jedem Check). +""" + +import subprocess +import sys +import json +import os +from datetime import datetime + +NOW = datetime.now().strftime("%Y-%m-%d %H:%M:%S") +STATE_FILE = "/tmp/bsn_watchdog_state.json" +ALERT_COOLDOWN = 900 # 15 Minuten zwischen Alerts + +# ── Telegram (optional) ── +TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN", "") +TELEGRAM_CHAT_ID = os.environ.get("ADMIN_TELEGRAM_CHAT_ID", "") +if not TELEGRAM_CHAT_ID: + # Fallback: aus Hermes Config lesen + try: + import yaml + with open(os.path.expanduser("~/.hermes/config.yaml")) as f: + cfg = yaml.safe_load(f) + home_channel = cfg.get("messaging", {}).get("telegram", {}).get("home_channel", "") + if home_channel: + TELEGRAM_CHAT_ID = home_channel + except: + pass + +def send_alert(msg: str): + """Sende Telegram-Alert (nur wenn Token + Chat ID vorhanden).""" + if not TELEGRAM_TOKEN or not TELEGRAM_CHAT_ID: + print(f"[alert] Telegram nicht konfiguriert: {msg}") + return + try: + url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage" + subprocess.run([ + "curl", "-s", "-X", "POST", url, + "-H", "Content-Type: application/json", + "-d", json.dumps({ + "chat_id": TELEGRAM_CHAT_ID, + "text": msg, + "parse_mode": "Markdown" + }) + ], timeout=10, capture_output=True) + except Exception as e: + print(f"[alert] Fehler: {e}") + +def load_state(): + try: + with open(STATE_FILE) as f: + return json.load(f) + except: + return {"last_alert": 0, "last_status": {}} + +def save_state(state): + with open(STATE_FILE, "w") as f: + json.dump(state, f) + +def check_port(port: int) -> bool: + """Prüft ob localhost:PORT antwortet.""" + try: + r = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"http://localhost:{port}/", "--max-time", "5"], + capture_output=True, text=True, timeout=6 + ) + return r.stdout.strip() == "200" + except: + return False + +def check_url(url: str) -> bool: + """Prüft ob externe URL antwortet.""" + try: + r = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + url, "--max-time", "10"], + capture_output=True, text=True, timeout=12 + ) + return r.stdout.strip() in ("200", "302", "301") + except: + return False + +def check_process(name_pattern: str) -> bool: + """Prüft ob Prozess läuft.""" + try: + r = subprocess.run( + ["pgrep", "-f", name_pattern], + capture_output=True, timeout=3 + ) + return r.returncode == 0 + except: + return False + +def try_recover(service: str) -> str: + """Versucht einen Service wiederherzustellen.""" + if service == "flask": + subprocess.Popen( + ["/home/hermes/venv/bin/python", "app.py"], + cwd="/home/hermes/workspace/bsn-chatbot", + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + return "Flask neugestartet" + elif service == "tunnel": + subprocess.Popen( + ["cloudflared", "tunnel", "--config", + "/home/hermes/.cloudflared/config.yml", "run"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + return "Tunnel neugestartet" + elif service == "article_server": + subprocess.Popen( + ["/home/hermes/venv/bin/python", "article_server.py"], + cwd="/home/hermes/workspace", + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + return "Article-Server neugestartet" + return "" + +# ── Checks ── +results = {} + +# 1. Flask +flask_process = check_process("python.*app.py") +flask_port = check_port(5002) +results["flask"] = { + "process": flask_process, + "port_5002": flask_port, + "status": "ok" if (flask_process and flask_port) else "down" +} + +# 2. Tunnel +tunnel_process = check_process("cloudflared.*tunnel") +tunnel_url = check_url("https://chat.datenhimmel.work/health") +results["tunnel"] = { + "process": tunnel_process, + "url": tunnel_url, + "status": "ok" if (tunnel_process and tunnel_url) else "down" +} + +# 3. Article Server (optional) +article_process = check_process("python.*article_server") +article_port = check_port(8890) +results["article_server"] = { + "process": article_process, + "port_8890": article_port, + "status": "ok" if (article_process and article_port) else "down" +} + +# 4. SQLite DB +try: + db_size = os.path.getsize("/home/hermes/workspace/bsn-chatbot/bsn_intake.db") + results["database"] = {"status": "ok", "size_mb": round(db_size/1024/1024, 1)} +except: + results["database"] = {"status": "down"} + +# 5. Speicher +try: + r = subprocess.run(["free", "-m"], capture_output=True, text=True, timeout=3) + mem_line = [l for l in r.stdout.split("\n") if "Mem:" in l] + if mem_line: + parts = mem_line[0].split() + results["memory"] = { + "total_mb": int(parts[1]), + "used_mb": int(parts[2]), + "available_mb": int(parts[6]), + "status": "ok" if int(parts[6]) > 500 else "warn" + } +except: + results["memory"] = {"status": "unknown"} + +# ── Status-Zusammenfassung ── +all_ok = all(r.get("status") == "ok" for r in results.values()) +down_services = [name for name, r in results.items() if r.get("status") == "down"] +warn_services = [name for name, r in results.items() if r.get("status") == "warn"] + +# ── State-Management ── +state = load_state() +prev_status = state.get("last_status", {}) +now_ts = int(datetime.now().timestamp()) + +# Status-Änderungen erkennen +changed = {} +for name, r in results.items(): + old = prev_status.get(name, "unknown") + new = r.get("status", "unknown") + if old != new: + changed[name] = f"{old} → {new}" + +# ── Ausgabe ── +status_line = "🟢" if all_ok else "🔴" if down_services else "🟡" +print(f"{status_line} BSN Watchdog {NOW}") +print(f" Flask: {'✅' if results['flask']['status']=='ok' else '❌'} (PID={'✓' if flask_process else '✗'} Port={'✓' if flask_port else '✗'})") +print(f" Tunnel: {'✅' if results['tunnel']['status']=='ok' else '❌'} (PID={'✓' if tunnel_process else '✗'} URL={'✓' if tunnel_url else '✗'})") +print(f" Article: {'✅' if results['article_server']['status']=='ok' else '⚪' if article_process else '—'} (PID={'✓' if article_process else '✗'} Port={'✓' if article_port else '✗'})") +print(f" DB: {'✅' if results['database']['status']=='ok' else '❌'} ({results['database'].get('size_mb','?')} MB)") +if "memory" in results: + mem = results["memory"] + print(f" RAM: {mem.get('available_mb','?')} MB frei / {mem.get('total_mb','?')} MB") + +# ── Alerts nur bei Änderung oder nach Cooldown ── +if down_services: + should_alert = False + if changed: + should_alert = True # Neuer Ausfall + elif now_ts - state.get("last_alert", 0) > ALERT_COOLDOWN: + should_alert = True # Periodische Erinnerung + + if should_alert: + alert_parts = [f"🔴 *BSN Watchdog — {len(down_services)} Service(s) down*\n"] + for svc in down_services: + r = results[svc] + details = [] + if "process" in r: details.append(f"Prozess={'✓' if r['process'] else '✗'}") + if "port_5002" in r: details.append(f"Port={'✓' if r['port_5002'] else '✗'}") + if "port_8890" in r: details.append(f"Port={'✓' if r['port_8890'] else '✗'}") + if "url" in r: details.append(f"URL={'✓' if r['url'] else '✗'}") + alert_parts.append(f"• *{svc}*: {', '.join(details)}") + + # Auto-Recovery + if svc in ("flask", "tunnel", "article_server"): + msg = try_recover(svc) + alert_parts.append(f" ↳ {msg}") + + if changed: + alert_parts.append(f"\n_Geändert: {', '.join(f'{k}: {v}' for k,v in changed.items())}_") + alert_parts.append(f"\n_{NOW}_") + + send_alert("\n".join(alert_parts)) + state["last_alert"] = now_ts + +elif all_ok and prev_status and any( + prev_status.get(n, "ok") != "ok" for n in results +): + # Recovery-Alert + recovered = [n for n in results if prev_status.get(n, "ok") != "ok"] + msg = f"🟢 *BSN Watchdog — Alle Services wieder online!*\nErholt: {', '.join(recovered)}\n_{NOW}_" + send_alert(msg) + state["last_alert"] = now_ts + +# ── State speichern ── +state["last_status"] = {name: r["status"] for name, r in results.items()} +save_state(state) + +# ── Exit-Code ── +if down_services: + print(f"\n❌ DOWN: {', '.join(down_services)}") + sys.exit(1) +elif warn_services: + print(f"\n⚠️ WARN: {', '.join(warn_services)}") + sys.exit(0) +else: + print("\n✅ Alle Systeme normal") + sys.exit(0)