danielkrause/BSN-Chatsystem
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sys-2 Edge-Queue — Signatur-Prüfung, Dedup, asynchrone Verarbeitung
Quality Gate / 🛡️ Lint + Type Check + Tests (push) Failing after 11m21s
Details

Browse Source 
- webhook_queue.py: WhatsApp X-Hub-Signature-256 + Telegram Webhook-Intake
- queue_handler.py: SQLite-basierte Queue mit Idempotenz (UNIQUE message_id)
- queue_worker.py: Asynchroner Worker für GPU-Verarbeitung
- app.py: Webhooks umgebaut → sofort 200 OK, Worker verarbeitet später
- Redis 8.0 läuft als Infrastruktur (Queue ist SQLite für Persistenz)
This commit is contained in:
Hermes Agent
2026-06-21 23:59:17 +02:00
parent 5bad339558
commit a5eb0e7d27
5 changed files with 834 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+7 -497
View File
@@ -1020,331 +1020,15 @@ def whatsapp_webhook():
return challenge, 200
return "Forbidden", 403
# POST: Incoming message
# POST: Incoming message → Queue (asynchrone Verarbeitung)
body = request.get_json(force=True, silent=True)
if not body:
return "OK", 200
try:
import threading
entries = body.get("entry", [])
db = get_db()
for entry in entries:
changes = entry.get("changes", [])
for change in changes:
value = change.get("value", {})
contacts = value.get("contacts", [{}])
messages = value.get("messages", [])
for msg in messages:
sender_id = msg.get("from", "unknown")
sender_name = contacts[0].get("profile", {}).get("name", "unknown")
msg_type = msg.get("type", "text")
content = ""
media_path = None
spiel_titel = None
category_override = None # set to "ausverkauft" for #out
if msg_type == "text":
content = msg.get("text", {}).get("body", "")
elif msg_type == "image":
content = msg.get("image", {}).get("caption", "[Bild ohne Text]")
media_id = msg.get("image", {}).get("id")
if media_id:
media_path, _ = download_media(media_id)
elif msg_type == "audio":
content = "[🎤 Sprachnachricht — Transkription läuft...]"
media_id = msg.get("audio", {}).get("id")
if media_id:
media_path, _ = download_media(media_id)
elif msg_type == "video":
content = msg.get("video", {}).get(
"caption", "[🎬 Video — Transkription läuft...]"
)
media_id = msg.get("video", {}).get("id")
if media_id:
media_path, _ = download_media(media_id)
elif msg_type == "document":
content = msg.get("document", {}).get(
"caption", "[Dokument]"
)
media_id = msg.get("document", {}).get("id")
if media_id:
media_path, _ = download_media(media_id)
else:
content = f"[{msg_type}]"
# #out detection: set category to "ausverkauft" and extract game title
if "#out" in content.lower():
category_override = "ausverkauft"
idx = content.lower().find("#out")
after = content[idx + 4:].strip()
if after:
spiel_titel = re.split(r'[.!?\n]| - | – | — | ist | war | in ', after, maxsplit=1)[0].strip()[:200]
# GDPR: "Meine Daten löschen" trigger
if msg_type == "text" and content.strip().lower() == "meine daten löschen":
_gdpr_delete_sender(db, sender_id)
if META_TOKEN:
try:
send_whatsapp_message(
sender_id,
"✅ Alle deine Daten wurden vollständig und endgültig vom Server gelöscht. "
"Es befinden sich keinerlei personenbezogene Daten mehr von dir bei uns. "
"Lediglich ein anonymer Vermerk über die erfolgte Löschung bleibt für "
"unsere Dokumentationspflicht.",
)
except Exception:
pass
continue
# FAQ-Trigger: "Keine Antwort" / "Warum keine Nachricht"
faq_keywords = [
"keine antwort", "bekomme keine", "keine nachricht",
"antwort kommt nicht", "warum keine", "keine rückmeldung",
"why no reply", "not receiving", "save contact", "nummer speichern"
]
if msg_type == "text" and any(kw in content.lower() for kw in faq_keywords):
if META_TOKEN:
try:
send_whatsapp_message(
sender_id,
"💡 *Tipp:* Damit unsere Antworten bei dir ankommen, "
"speichere bitte unsere Nummer in deinem Adressbuch. "
"WhatsApp filtert Nachrichten von unbekannten Business-Accounts "
"auf manchen Handys leider automatisch heraus.\n\n"
"Sobald du uns als Kontakt gespeichert hast, klappt's! ✅"
)
except Exception:
pass
continue
# ── Nutzer-Registrierung per WhatsApp ────────────────
pending_codes = getattr(app, '_pending_codes', {})
if msg_type == "text":
clean = content.strip().lower()
# "registrieren" → Code senden
if clean == "registrieren":
import random
code = str(random.randint(100000, 999999))
pending_codes[sender_id] = code
app._pending_codes = pending_codes
if META_TOKEN:
try:
send_whatsapp_message(sender_id,
f"🔐 Dein Verifikationscode: *{code}*\n\n"
f"Antworte einfach mit: code {code}")
except Exception:
pass
continue
# "code 123456" → verifizieren + User anlegen
if clean.startswith("code ") and len(clean) >= 10:
entered = clean[5:].strip()
if sender_id in pending_codes and pending_codes[sender_id] == entered:
del pending_codes[sender_id]
app._pending_codes = pending_codes
# User anlegen — phone_hash statt Klartext
existing = _lookup_user_by_phone(db, sender_id)
if not existing:
phone_hash_val, phone_last4_val = _hash_phone(sender_id)
db.execute(
"INSERT INTO users (phone, name, verified, phone_hash, phone_last4) VALUES (?,?,1,?,?)",
(phone_last4_val, sender_name, phone_hash_val, phone_last4_val),
)
db.commit()
if META_TOKEN:
try:
send_whatsapp_message(sender_id,
"✅ Du bist jetzt registriert! Dein Profil: "
"https://chat.datenhimmel.work/profile")
except Exception:
pass
else:
if META_TOKEN:
try:
send_whatsapp_message(sender_id,
"❌ Falscher Code. Schick \"registrieren\" für einen neuen.")
except Exception:
pass
continue
# ── Cross-Channel: User-Profil zuordnen (via phone_hash) ──
user_row = _lookup_user_by_phone(db, sender_id)
# Check for existing thread from this sender (5-min window)
recent = db.execute(
"""SELECT id, content, media_path, halle FROM submissions
WHERE channel='whatsapp' AND sender_id=? AND status != 'gdpr_deleted'
ORDER BY created_at DESC LIMIT 1""",
(sender_id,),
).fetchone()
is_new_user = db.execute(
"SELECT COUNT(*) FROM submissions WHERE channel='whatsapp' AND sender_id=? AND status != 'gdpr_deleted'",
(sender_id,),
).fetchone()[0] == 0
THREAD_WINDOW_MINUTES = 5
thread_found = False
if recent:
age_check = db.execute(
"SELECT (strftime('%s','now') - strftime('%s',created_at)) < ? FROM submissions WHERE id=?",
(THREAD_WINDOW_MINUTES * 60, recent["id"]),
).fetchone()
if age_check and age_check[0]:
thread_found = True
if thread_found:
# Append to existing thread
prefix = {
"text": "", "image": "\n[🖼️ Bild] ",
"audio": "\n[🎤 Sprachnachricht]",
"video": "\n[🎬 Video] ", "document": "\n[📎 Dokument] ",
}.get(msg_type, f"\n[{msg_type}] ")
new_content = (recent["content"] or "") + prefix + content
# Re-extract halle from combined content
new_halle = extract_halle(new_content) or recent["halle"]
db.execute(
"UPDATE submissions SET content=?, halle=?, created_at=CURRENT_TIMESTAMP WHERE id=?",
(new_content, new_halle, recent["id"]),
)
# Check for #out in appended content — update category + spiel_titel
if "#out" in content.lower():
out_title = None
idx = content.lower().find("#out")
after = content[idx + 4:].strip()
if after:
out_title = re.split(r'[.!?\n]| - | – | — | ist | war | in ', after, maxsplit=1)[0].strip()[:200]
db.execute(
"UPDATE submissions SET category=?, spiel_titel=? WHERE id=?",
("ausverkauft", out_title, recent["id"]),
)
# Auto-mark matching neuheiten as sold out
if out_title:
mark_ausverkauft_in_neuheiten(db, out_title)
# Merge media paths (keep all, separated by ||)
if media_path:
merged_media = media_path
if recent["media_path"]:
merged_media = recent["media_path"] + "||" + media_path
db.execute(
"UPDATE submissions SET media_path=? WHERE id=?",
(merged_media, recent["id"]),
)
else:
# New thread
halle = extract_halle(content)
cat = category_override or None
db.execute(
"""INSERT INTO submissions
(channel, sender_id, sender_name, type, content, media_path, status, halle, category, spiel_titel, user_id)
VALUES (?, ?, ?, ?, ?, ?, 'new', ?, ?, ?, ?)""",
("whatsapp", sender_id, sender_name, msg_type, content, media_path, halle, cat, spiel_titel,
user_row["id"] if user_row else None),
)
# Auto-mark matching neuheiten as sold out
if cat == "ausverkauft" and spiel_titel:
mark_ausverkauft_in_neuheiten(db, spiel_titel)
# ── Admin-Benachrichtigung: Neue Einsendung ──────────
sub_id = recent["id"] if thread_found else db.execute(
"SELECT last_insert_rowid()"
).fetchone()[0]
content_preview = content.replace("\n", " ")[:80]
type_emoji = {"text": "💬", "image": "🖼️", "audio": "🎤", "video": "🎬"}.get(msg_type, "📎")
notify_admin(
f"{type_emoji} {sender_name}: {content_preview}\n"
f"→ https://chat.datenhimmel.work/admin/{sub_id}"
)
# Auto-reply: only on NEW threads, not appends
if not thread_found and META_TOKEN:
try:
if is_new_user:
reply = (
f"🎲 Hey {sender_name}! Cool, dass du da bist — willkommen in der "
f"brettspiel-news.de Community!\n\n"
f"Schick uns einfach, was dich bewegt: 💬 Text, 🎤 Sprachnachrichten, "
f"🖼️ Fotos oder 🎬 Videos — alles rund um Brettspiele. Wir schauen uns alles an!\n\n"
f"✍️ Nenn uns deinen Vornamen, wenn du magst — das ist deine Einwilligung, "
f"dass wir dich als Quelle nennen, falls wir deinen Beitrag verwenden. "
f"Ohne Namen schreiben wir „anonyme Quelle\".\n\n"
f"💡 Speicher unsere Nummer im Adressbuch, "
f"damit unsere Antworten sicher ankommen.\n\n"
f"🔒 „Meine Daten löschen\" genügt — und alles ist sofort und vollständig weg."
)
send_whatsapp_message(sender_id, reply)
# Thorsten voice: same text but without emojis
tts_text = re.sub(r'[^\x00-\x7F\u00C0-\u00FF\u0100-\u017F\u0180-\u024F\u1E00-\u1EFF\u2018-\u201F\u2013\u2014\u2026\u00A0\u00AB\u00BB\u20AC]', '', reply)
tts_text = re.sub(r' +', ' ', tts_text).strip()
audio_path = generate_thorsten_tts(tts_text)
if audio_path:
media_id = upload_whatsapp_media(audio_path)
if media_id:
send_whatsapp_audio(sender_id, media_id)
else:
reply = (
f"📨 Danke {sender_name}! Ist angekommen — "
f"wir kümmern uns drum."
)
send_whatsapp_message(sender_id, reply)
except Exception:
pass
# Transcribe audio in background (non-blocking)
if msg_type == "audio" and media_path:
sub_id = recent["id"] if thread_found else db.execute(
"SELECT last_insert_rowid()"
).fetchone()[0]
threading.Thread(
target=_transcribe_bg,
args=(DB_PATH, sub_id, media_path),
daemon=True,
).start()
# Transcribe video in background
if msg_type == "video" and media_path:
sub_id = recent["id"] if thread_found else db.execute(
"SELECT last_insert_rowid()"
).fetchone()[0]
threading.Thread(
target=_transcribe_video_bg,
args=(DB_PATH, sub_id, media_path),
daemon=True,
).start()
# Auto-summarize text/image submissions (LLM hall + summary)
if msg_type in ("text", "image") and not thread_found and content and len(content.strip()) > 20:
sub_id = db.execute("SELECT last_insert_rowid()").fetchone()[0]
# Only if regex didn't already find a halle
if not halle:
threading.Thread(
target=auto_process_bg,
args=(DB_PATH, sub_id),
daemon=True,
).start()
# LLM-Triage: 3-Stufen-Klassifikation (nur für neue Threads)
if not thread_found:
sub_id = db.execute("SELECT last_insert_rowid()").fetchone()[0]
threading.Thread(
target=_run_triage_async,
args=(sub_id,),
daemon=True,
).start()
db.commit()
except Exception as e:
print(f"[webhook error] {e}", file=sys.stderr)
return "OK", 200
return "OK", 200
from webhook_queue import whatsapp_intake
return whatsapp_intake(request, db)
@app.route("/admin")
def admin_dashboard():
"""Internal admin dashboard — filterable table of all submissions."""
db = get_db()
channel_filter = request.args.get("channel", "")
@@ -5392,189 +5076,15 @@ def telegram_webhook():
return jsonify({"status": "off", "reason": "TELEGRAM_TOKEN not set"}), 500
# POST: Incoming message
# POST: Incoming message → Queue (asynchrone Verarbeitung)
body = request.get_json(force=True, silent=True)
if not body or not TELEGRAM_TOKEN:
return "OK", 200
try:
import threading
msg = body.get("message", {})
if not msg:
return "OK", 200
from webhook_queue import telegram_intake
db = get_db()
return telegram_intake(request, db)
chat = msg.get("chat", {})
sender_id = str(chat.get("id", "unknown"))
sender_name = msg.get("from", {}).get("first_name", "") or ""
last_name = msg.get("from", {}).get("last_name", "")
if last_name:
sender_name = "{} {}".format(sender_name, last_name).strip()
if not sender_name:
sender_name = sender_id
msg_type = "text"
content = ""
media_path = None
file_id = None
if "text" in msg:
msg_type = "text"
content = msg["text"]
elif "photo" in msg:
msg_type = "image"
content = msg.get("caption", "[Bild ohne Text]")
file_id = msg["photo"][-1]["file_id"]
elif "video" in msg:
msg_type = "video"
content = msg.get("caption", "[🎬 Video — Transkription läuft...]")
file_id = msg["video"]["file_id"]
elif "voice" in msg:
msg_type = "audio"
content = "[🎤 Sprachnachricht — Transkription läuft...]"
file_id = msg["voice"]["file_id"]
elif "audio" in msg:
msg_type = "audio"
content = msg.get("caption", "[🎤 Audio]")
file_id = msg["audio"]["file_id"]
elif "document" in msg:
msg_type = "document"
content = msg.get("caption", "[📎 Dokument]")
file_id = msg["document"]["file_id"]
elif "sticker" in msg:
content = "[Sticker: {}]".format(msg['sticker'].get('emoji', ''))
msg_type = "document"
elif "location" in msg:
lat = msg["location"]["latitude"]
lon = msg["location"]["longitude"]
content = "[📍 Standort: {}, {}]".format(lat, lon)
msg_type = "text"
else:
content = "[Unbekannter Nachrichtentyp]"
msg_type = "text"
if file_id:
media_path = _telegram_download_file(file_id)
# GDPR delete trigger
if msg_type == "text" and content.strip().lower() == "meine daten löschen":
_gdpr_delete_sender(get_db(), sender_id)
if TELEGRAM_TOKEN:
import requests as _r
try:
_r.post(
"{}/sendMessage".format(TELEGRAM_API),
json={"chat_id": int(sender_id),
"text": "✅ Alle deine Daten wurden vollständig und endgültig vom Server gelöscht. "
"Es befinden sich keinerlei personenbezogene Daten mehr von dir bei uns."},
timeout=10,
)
except Exception:
pass
return "OK", 200
db = get_db()
spiel_titel = None
category_override = None
if "#out" in content.lower():
category_override = "ausverkauft"
idx = content.lower().find("#out")
after = content[idx + 4:].strip()
if after:
spiel_titel = re.split(r'[.!?\n]| - | – | — | ist | war | in ', after, maxsplit=1)[0].strip()[:200]
halle = extract_halle(content)
THREAD_WINDOW_MINUTES = 5
recent = db.execute(
"""SELECT id, content, media_path, halle FROM submissions
WHERE channel='telegram' AND sender_id=? AND status != 'gdpr_deleted'
ORDER BY created_at DESC LIMIT 1""",
(sender_id,),
).fetchone()
is_new_user = db.execute(
"SELECT COUNT(*) FROM submissions WHERE channel='telegram' AND sender_id=? AND status != 'gdpr_deleted'",
(sender_id,),
).fetchone()[0] == 0
thread_found = False
if recent:
age_check = db.execute(
"SELECT (strftime('%s','now') - strftime('%s',created_at)) < ? FROM submissions WHERE id=?",
(THREAD_WINDOW_MINUTES * 60, recent["id"]),
).fetchone()
if age_check and age_check[0]:
thread_found = True
if thread_found:
prefix = {
"text": "", "image": "\n[🖼️ Bild] ",
"audio": "\n[🎤 Sprachnachricht]", "video": "\n[🎬 Video] ",
"document": "\n[📎 Dokument] ",
}.get(msg_type, "\n[{}] ".format(msg_type))
new_content = (recent["content"] or "") + prefix + content
new_halle = extract_halle(new_content) or recent["halle"]
db.execute(
"UPDATE submissions SET content=?, halle=?, created_at=CURRENT_TIMESTAMP WHERE id=?",
(new_content, new_halle, recent["id"]),
)
if "#out" in content.lower():
out_title = None
idx2 = content.lower().find("#out")
after2 = content[idx2 + 4:].strip()
if after2:
out_title = re.split(r'[.!?\n]| - | – | — | ist | war | in ', after2, maxsplit=1)[0].strip()[:200]
db.execute("UPDATE submissions SET category=?, spiel_titel=? WHERE id=?",
("ausverkauft", out_title, recent["id"]))
if out_title:
mark_ausverkauft_in_neuheiten(db, out_title)
if media_path:
merged = media_path
if recent["media_path"]:
merged = recent["media_path"] + "||" + media_path
db.execute("UPDATE submissions SET media_path=? WHERE id=?", (merged, recent["id"]))
else:
cat = category_override or None
db.execute(
"""INSERT INTO submissions
(channel, sender_id, sender_name, type, content, media_path, status, halle, category, spiel_titel)
VALUES (?, ?, ?, ?, ?, ?, 'new', ?, ?, ?)""",
("telegram", sender_id, sender_name, msg_type, content, media_path, halle, cat, spiel_titel),
)
if cat == "ausverkauft" and spiel_titel:
mark_ausverkauft_in_neuheiten(db, spiel_titel)
if not thread_found:
_telegram_autoreply(int(sender_id), sender_name, is_new_user)
if msg_type == "audio" and media_path:
import threading
sub_id = recent["id"] if thread_found else db.execute("SELECT last_insert_rowid()").fetchone()[0]
threading.Thread(target=_transcribe_bg, args=(DB_PATH, sub_id, media_path), daemon=True).start()
if msg_type == "video" and media_path:
import threading
sub_id = recent["id"] if thread_found else db.execute("SELECT last_insert_rowid()").fetchone()[0]
threading.Thread(target=_transcribe_video_bg, args=(DB_PATH, sub_id, media_path), daemon=True).start()
if msg_type in ("text", "image") and not thread_found and content and len(content.strip()) > 20:
import threading
sub_id = db.execute("SELECT last_insert_rowid()").fetchone()[0]
if not halle:
threading.Thread(target=auto_process_bg, args=(DB_PATH, sub_id), daemon=True).start()
# LLM-Triage: 3-Stufen-Klassifikation (nur für neue Threads)
if not thread_found:
import threading
sub_id = db.execute("SELECT last_insert_rowid()").fetchone()[0]
threading.Thread(target=_run_triage_async, args=(sub_id,), daemon=True).start()
db.commit()
except Exception as e:
print("[telegram webhook error] {}".format(e), file=sys.stderr)
return "OK", 200
return "OK", 200
if __name__ == "__main__":
import threading