BSN-Chatsystem/scripts/watchdog.py

#!/usr/bin/env python3
"""BSN Watchdog — Überwacht Flask + Cloudflare Tunnel + Article Server.
   Läuft als Cron-Job alle 5 Minuten.
   Sendet Alert per Telegram nur bei Zustands-ÄNDERUNG (nicht bei jedem Check).
"""

import subprocess
import sys
import json
import os
from datetime import datetime

NOW = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
STATE_FILE = "/tmp/bsn_watchdog_state.json"
ALERT_COOLDOWN = 900  # 15 Minuten zwischen Alerts

# ── Telegram (optional) ──
TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN", "")
TELEGRAM_CHAT_ID = os.environ.get("ADMIN_TELEGRAM_CHAT_ID", "")
if not TELEGRAM_CHAT_ID:
    # Fallback: aus Hermes Config lesen
    try:
        import yaml
        with open(os.path.expanduser("~/.hermes/config.yaml")) as f:
            cfg = yaml.safe_load(f)
        home_channel = cfg.get("messaging", {}).get("telegram", {}).get("home_channel", "")
        if home_channel:
            TELEGRAM_CHAT_ID = home_channel
    except:
        pass

def send_alert(msg: str):
    """Sende Telegram-Alert (nur wenn Token + Chat ID vorhanden)."""
    if not TELEGRAM_TOKEN or not TELEGRAM_CHAT_ID:
        print(f"[alert] Telegram nicht konfiguriert: {msg}")
        return
    try:
        url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage"
        subprocess.run([
            "curl", "-s", "-X", "POST", url,
            "-H", "Content-Type: application/json",
            "-d", json.dumps({
                "chat_id": TELEGRAM_CHAT_ID,
                "text": msg,
                "parse_mode": "Markdown"
            })
        ], timeout=10, capture_output=True)
    except Exception as e:
        print(f"[alert] Fehler: {e}")

def load_state():
    try:
        with open(STATE_FILE) as f:
            return json.load(f)
    except:
        return {"last_alert": 0, "last_status": {}}

def save_state(state):
    with open(STATE_FILE, "w") as f:
        json.dump(state, f)

def check_port(port: int) -> bool:
    """Prüft ob localhost:PORT antwortet."""
    try:
        r = subprocess.run(
            ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
             f"http://localhost:{port}/", "--max-time", "5"],
            capture_output=True, text=True, timeout=6
        )
        return r.stdout.strip() == "200"
    except:
        return False

def check_url(url: str) -> bool:
    """Prüft ob externe URL antwortet."""
    try:
        r = subprocess.run(
            ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
             url, "--max-time", "10"],
            capture_output=True, text=True, timeout=12
        )
        return r.stdout.strip() in ("200", "302", "301")
    except:
        return False

def check_process(name_pattern: str) -> bool:
    """Prüft ob Prozess läuft."""
    try:
        r = subprocess.run(
            ["pgrep", "-f", name_pattern],
            capture_output=True, timeout=3
        )
        return r.returncode == 0
    except:
        return False

def try_recover(service: str) -> str:
    """Versucht einen Service wiederherzustellen."""
    if service == "flask":
        subprocess.Popen(
            ["/home/hermes/venv/bin/python", "app.py"],
            cwd="/home/hermes/workspace/bsn-chatbot",
            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
        )
        return "Flask neugestartet"
    elif service == "tunnel":
        subprocess.Popen(
            ["cloudflared", "tunnel", "--config",
             "/home/hermes/.cloudflared/config.yml", "run"],
            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
        )
        return "Tunnel neugestartet"
    elif service == "article_server":
        subprocess.Popen(
            ["/home/hermes/venv/bin/python", "article_server.py"],
            cwd="/home/hermes/workspace",
            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
        )
        return "Article-Server neugestartet"
    return ""

# ── Checks ──
results = {}

# 1. Flask
flask_process = check_process("python.*app.py")
flask_port = check_port(5002)
results["flask"] = {
    "process": flask_process,
    "port_5002": flask_port,
    "status": "ok" if (flask_process and flask_port) else "down"
}

# 2. Tunnel
tunnel_process = check_process("cloudflared.*tunnel")
tunnel_url = check_url("https://chat.datenhimmel.work/health")
results["tunnel"] = {
    "process": tunnel_process,
    "url": tunnel_url,
    "status": "ok" if (tunnel_process and tunnel_url) else "down"
}

# 3. Article Server (optional)
article_process = check_process("python.*article_server")
article_port = check_port(8890)
results["article_server"] = {
    "process": article_process,
    "port_8890": article_port,
    "status": "ok" if (article_process and article_port) else "down"
}

# 4. SQLite DB
try:
    db_size = os.path.getsize("/home/hermes/workspace/bsn-chatbot/bsn_intake.db")
    results["database"] = {"status": "ok", "size_mb": round(db_size/1024/1024, 1)}
except:
    results["database"] = {"status": "down"}

# 5. Speicher
try:
    r = subprocess.run(["free", "-m"], capture_output=True, text=True, timeout=3)
    mem_line = [l for l in r.stdout.split("\n") if "Mem:" in l]
    if mem_line:
        parts = mem_line[0].split()
        results["memory"] = {
            "total_mb": int(parts[1]),
            "used_mb": int(parts[2]),
            "available_mb": int(parts[6]),
            "status": "ok" if int(parts[6]) > 500 else "warn"
        }
except:
    results["memory"] = {"status": "unknown"}

# ── Status-Zusammenfassung ──
all_ok = all(r.get("status") == "ok" for r in results.values())
down_services = [name for name, r in results.items() if r.get("status") == "down"]
warn_services = [name for name, r in results.items() if r.get("status") == "warn"]

# ── State-Management ──
state = load_state()
prev_status = state.get("last_status", {})
now_ts = int(datetime.now().timestamp())

# Status-Änderungen erkennen
changed = {}
for name, r in results.items():
    old = prev_status.get(name, "unknown")
    new = r.get("status", "unknown")
    if old != new:
        changed[name] = f"{old} → {new}"

# ── Ausgabe ──
status_line = "🟢" if all_ok else "🔴" if down_services else "🟡"
print(f"{status_line} BSN Watchdog {NOW}")
print(f"   Flask:      {'✅' if results['flask']['status']=='ok' else '❌'} (PID={'✓' if flask_process else '✗'} Port={'✓' if flask_port else '✗'})")
print(f"   Tunnel:     {'✅' if results['tunnel']['status']=='ok' else '❌'} (PID={'✓' if tunnel_process else '✗'} URL={'✓' if tunnel_url else '✗'})")
print(f"   Article:    {'✅' if results['article_server']['status']=='ok' else '⚪' if article_process else '—'} (PID={'✓' if article_process else '✗'} Port={'✓' if article_port else '✗'})")
print(f"   DB:         {'✅' if results['database']['status']=='ok' else '❌'} ({results['database'].get('size_mb','?')} MB)")
if "memory" in results:
    mem = results["memory"]
    print(f"   RAM:        {mem.get('available_mb','?')} MB frei / {mem.get('total_mb','?')} MB")

# ── Alerts nur bei Änderung oder nach Cooldown ──
if down_services:
    should_alert = False
    if changed:
        should_alert = True  # Neuer Ausfall
    elif now_ts - state.get("last_alert", 0) > ALERT_COOLDOWN:
        should_alert = True  # Periodische Erinnerung

    if should_alert:
        alert_parts = [f"🔴 *BSN Watchdog — {len(down_services)} Service(s) down*\n"]
        for svc in down_services:
            r = results[svc]
            details = []
            if "process" in r: details.append(f"Prozess={'✓' if r['process'] else '✗'}")
            if "port_5002" in r: details.append(f"Port={'✓' if r['port_5002'] else '✗'}")
            if "port_8890" in r: details.append(f"Port={'✓' if r['port_8890'] else '✗'}")
            if "url" in r: details.append(f"URL={'✓' if r['url'] else '✗'}")
            alert_parts.append(f"• *{svc}*: {', '.join(details)}")

            # Auto-Recovery
            if svc in ("flask", "tunnel", "article_server"):
                msg = try_recover(svc)
                alert_parts.append(f"  ↳ {msg}")

        if changed:
            alert_parts.append(f"\n_Geändert: {', '.join(f'{k}: {v}' for k,v in changed.items())}_")
        alert_parts.append(f"\n_{NOW}_")
        
        send_alert("\n".join(alert_parts))
        state["last_alert"] = now_ts

elif all_ok and prev_status and any(
    prev_status.get(n, "ok") != "ok" for n in results
):
    # Recovery-Alert
    recovered = [n for n in results if prev_status.get(n, "ok") != "ok"]
    msg = f"🟢 *BSN Watchdog — Alle Services wieder online!*\nErholt: {', '.join(recovered)}\n_{NOW}_"
    send_alert(msg)
    state["last_alert"] = now_ts

# ── State speichern ──
state["last_status"] = {name: r["status"] for name, r in results.items()}
save_state(state)

# ── Exit-Code ──
if down_services:
    print(f"\n❌ DOWN: {', '.join(down_services)}")
    sys.exit(1)
elif warn_services:
    print(f"\n⚠️ WARN: {', '.join(warn_services)}")
    sys.exit(0)
else:
    print("\n✅ Alle Systeme normal")
    sys.exit(0)