#!/usr/bin/env python3 """BSN Watchdog — Überwacht Flask + Cloudflare Tunnel + Article Server. Läuft als Cron-Job alle 5 Minuten. Sendet Alert per Telegram nur bei Zustands-ÄNDERUNG (nicht bei jedem Check). """ import subprocess import sys import json import os from datetime import datetime NOW = datetime.now().strftime("%Y-%m-%d %H:%M:%S") STATE_FILE = "/tmp/bsn_watchdog_state.json" ALERT_COOLDOWN = 900 # 15 Minuten zwischen Alerts # ── Telegram (optional) ── TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN", "") TELEGRAM_CHAT_ID = os.environ.get("ADMIN_TELEGRAM_CHAT_ID", "") if not TELEGRAM_CHAT_ID: # Fallback: aus Hermes Config lesen try: import yaml with open(os.path.expanduser("~/.hermes/config.yaml")) as f: cfg = yaml.safe_load(f) home_channel = cfg.get("messaging", {}).get("telegram", {}).get("home_channel", "") if home_channel: TELEGRAM_CHAT_ID = home_channel except: pass def send_alert(msg: str): """Sende Telegram-Alert (nur wenn Token + Chat ID vorhanden).""" if not TELEGRAM_TOKEN or not TELEGRAM_CHAT_ID: print(f"[alert] Telegram nicht konfiguriert: {msg}") return try: url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage" subprocess.run([ "curl", "-s", "-X", "POST", url, "-H", "Content-Type: application/json", "-d", json.dumps({ "chat_id": TELEGRAM_CHAT_ID, "text": msg, "parse_mode": "Markdown" }) ], timeout=10, capture_output=True) except Exception as e: print(f"[alert] Fehler: {e}") def load_state(): try: with open(STATE_FILE) as f: return json.load(f) except: return {"last_alert": 0, "last_status": {}} def save_state(state): with open(STATE_FILE, "w") as f: json.dump(state, f) def check_port(port: int) -> bool: """Prüft ob localhost:PORT antwortet.""" try: r = subprocess.run( ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", f"http://localhost:{port}/", "--max-time", "5"], capture_output=True, text=True, timeout=6 ) return r.stdout.strip() == "200" except: return False def check_url(url: str) -> bool: """Prüft ob externe URL antwortet.""" try: r = subprocess.run( ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", url, "--max-time", "10"], capture_output=True, text=True, timeout=12 ) return r.stdout.strip() in ("200", "302", "301") except: return False def check_process(name_pattern: str) -> bool: """Prüft ob Prozess läuft.""" try: r = subprocess.run( ["pgrep", "-f", name_pattern], capture_output=True, timeout=3 ) return r.returncode == 0 except: return False def try_recover(service: str) -> str: """Versucht einen Service wiederherzustellen.""" if service == "flask": subprocess.Popen( ["/home/hermes/venv/bin/python", "app.py"], cwd="/home/hermes/workspace/bsn-chatbot", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return "Flask neugestartet" elif service == "tunnel": subprocess.Popen( ["cloudflared", "tunnel", "--config", "/home/hermes/.cloudflared/config.yml", "run"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return "Tunnel neugestartet" elif service == "article_server": subprocess.Popen( ["/home/hermes/venv/bin/python", "article_server.py"], cwd="/home/hermes/workspace", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return "Article-Server neugestartet" return "" # ── Checks ── results = {} # 1. Flask flask_process = check_process("python.*app.py") flask_port = check_port(5002) results["flask"] = { "process": flask_process, "port_5002": flask_port, "status": "ok" if (flask_process and flask_port) else "down" } # 2. Tunnel tunnel_process = check_process("cloudflared.*tunnel") tunnel_url = check_url("https://chat.datenhimmel.work/health") results["tunnel"] = { "process": tunnel_process, "url": tunnel_url, "status": "ok" if (tunnel_process and tunnel_url) else "down" } # 3. Article Server (optional) article_process = check_process("python.*article_server") article_port = check_port(8890) results["article_server"] = { "process": article_process, "port_8890": article_port, "status": "ok" if (article_process and article_port) else "down" } # 4. SQLite DB try: db_size = os.path.getsize("/home/hermes/workspace/bsn-chatbot/bsn_intake.db") results["database"] = {"status": "ok", "size_mb": round(db_size/1024/1024, 1)} except: results["database"] = {"status": "down"} # 5. Speicher try: r = subprocess.run(["free", "-m"], capture_output=True, text=True, timeout=3) mem_line = [l for l in r.stdout.split("\n") if "Mem:" in l] if mem_line: parts = mem_line[0].split() results["memory"] = { "total_mb": int(parts[1]), "used_mb": int(parts[2]), "available_mb": int(parts[6]), "status": "ok" if int(parts[6]) > 500 else "warn" } except: results["memory"] = {"status": "unknown"} # ── Status-Zusammenfassung ── all_ok = all(r.get("status") == "ok" for r in results.values()) down_services = [name for name, r in results.items() if r.get("status") == "down"] warn_services = [name for name, r in results.items() if r.get("status") == "warn"] # ── State-Management ── state = load_state() prev_status = state.get("last_status", {}) now_ts = int(datetime.now().timestamp()) # Status-Änderungen erkennen changed = {} for name, r in results.items(): old = prev_status.get(name, "unknown") new = r.get("status", "unknown") if old != new: changed[name] = f"{old} → {new}" # ── Ausgabe ── status_line = "🟢" if all_ok else "🔴" if down_services else "🟡" print(f"{status_line} BSN Watchdog {NOW}") print(f" Flask: {'✅' if results['flask']['status']=='ok' else '❌'} (PID={'✓' if flask_process else '✗'} Port={'✓' if flask_port else '✗'})") print(f" Tunnel: {'✅' if results['tunnel']['status']=='ok' else '❌'} (PID={'✓' if tunnel_process else '✗'} URL={'✓' if tunnel_url else '✗'})") print(f" Article: {'✅' if results['article_server']['status']=='ok' else '⚪' if article_process else '—'} (PID={'✓' if article_process else '✗'} Port={'✓' if article_port else '✗'})") print(f" DB: {'✅' if results['database']['status']=='ok' else '❌'} ({results['database'].get('size_mb','?')} MB)") if "memory" in results: mem = results["memory"] print(f" RAM: {mem.get('available_mb','?')} MB frei / {mem.get('total_mb','?')} MB") # ── Alerts nur bei Änderung oder nach Cooldown ── if down_services: should_alert = False if changed: should_alert = True # Neuer Ausfall elif now_ts - state.get("last_alert", 0) > ALERT_COOLDOWN: should_alert = True # Periodische Erinnerung if should_alert: alert_parts = [f"🔴 *BSN Watchdog — {len(down_services)} Service(s) down*\n"] for svc in down_services: r = results[svc] details = [] if "process" in r: details.append(f"Prozess={'✓' if r['process'] else '✗'}") if "port_5002" in r: details.append(f"Port={'✓' if r['port_5002'] else '✗'}") if "port_8890" in r: details.append(f"Port={'✓' if r['port_8890'] else '✗'}") if "url" in r: details.append(f"URL={'✓' if r['url'] else '✗'}") alert_parts.append(f"• *{svc}*: {', '.join(details)}") # Auto-Recovery if svc in ("flask", "tunnel", "article_server"): msg = try_recover(svc) alert_parts.append(f" ↳ {msg}") if changed: alert_parts.append(f"\n_Geändert: {', '.join(f'{k}: {v}' for k,v in changed.items())}_") alert_parts.append(f"\n_{NOW}_") send_alert("\n".join(alert_parts)) state["last_alert"] = now_ts elif all_ok and prev_status and any( prev_status.get(n, "ok") != "ok" for n in results ): # Recovery-Alert recovered = [n for n in results if prev_status.get(n, "ok") != "ok"] msg = f"🟢 *BSN Watchdog — Alle Services wieder online!*\nErholt: {', '.join(recovered)}\n_{NOW}_" send_alert(msg) state["last_alert"] = now_ts # ── State speichern ── state["last_status"] = {name: r["status"] for name, r in results.items()} save_state(state) # ── Exit-Code ── if down_services: print(f"\n❌ DOWN: {', '.join(down_services)}") sys.exit(1) elif warn_services: print(f"\n⚠️ WARN: {', '.join(warn_services)}") sys.exit(0) else: print("\n✅ Alle Systeme normal") sys.exit(0)