danielkrause/BSN-Chatsystem
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: OOM-safe SQL filtering + relevance scoring for /api/chat

Browse Source 
Problem: On every chat request, ALL 1015 aussteller + 1861 neuheiten rows
were loaded into Python (2876 rows × full-text processing). Multiple
concurrent requests caused OOM kills (exit 137).

Fix:
- SQL LIKE filtering with parameterized queries (WHERE ... OR ... LIMIT 30)
- Relevance scoring in SQL: longer search terms = higher weight
- Tokenization: re.findall strips punctuation ('kosmos?' → 'kosmos')
- Stopword filtering removes noise ('die', 'hat', etc.)
- Removed duplicate 'gibt' in stopwords

Verified: Kosmos, Asmodee (6 Stände), Pegasus (3 Stände), Flip & Write
This commit is contained in:
Hermes Agent
2026-06-18 14:56:37 +02:00
parent ce25870960
commit bdd7ebb006
1 changed files with 74 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+67 -40
View File
@@ -3105,23 +3105,55 @@ def api_chat():
+ (f" | von {name}" if name else '')
)
# Also search aussteller table for matching exhibitors
search_terms = user_message.lower().split()
# Also search aussteller table using SQL LIKE filters (OOM-safe)
# Extract alphanumeric+umlaut tokens (strip punctuation like '?', '!', ',')
search_terms = re.findall(r'[a-z0-9äöüß]+', user_message.lower())
# Filter stopwords to reduce noise
stopwords = {'die', 'der', 'das', 'und', 'ist', 'ein', 'eine', 'den', 'dem', 'des',
'mit', 'auf', 'für', 'von', 'zu', 'im', 'in', 'an', 'es', 'nicht',
'ich', 'wir', 'sie', 'er', 'was', 'wo', 'wie', 'welche', 'welcher',
'welches', 'gibt', 'noch', 'mehr', 'alle', 'oder', 'auch'}
meaningful_terms = [t for t in search_terms if t not in stopwords and len(t) > 1]
# Build SQL LIKE filter for aussteller with relevance scoring
aussteller_rows = []
if meaningful_terms:
# Build score expression: longer terms = higher weight
score_parts = []
like_clauses = []
params = []
for t in meaningful_terms:
term_len = len(t)
score_parts.append(
f"(CASE WHEN LOWER(name) LIKE ? OR LOWER(halle) LIKE ? OR LOWER(stand) LIKE ? "
f"OR LOWER(land) LIKE ? OR LOWER(info) LIKE ? OR LOWER(themen) LIKE ? THEN {term_len} ELSE 0 END)"
)
like_clauses.append(
"(LOWER(name) LIKE ? OR LOWER(halle) LIKE ? OR LOWER(stand) LIKE ? "
"OR LOWER(land) LIKE ? OR LOWER(info) LIKE ? OR LOWER(themen) LIKE ?)"
)
# Push params: first 6 for scoring, then 6 for WHERE
params.extend([f'%{t}%'] * 6)
# Re-add WHERE params after score params
params.extend(params[:]) # duplicate for WHERE clause
score_expr = ' + '.join(score_parts)
sql = (
f"SELECT name, halle, stand, land, info, web, themen, ({score_expr}) AS relevance "
"FROM aussteller "
f"WHERE {' OR '.join(like_clauses)} "
"ORDER BY relevance DESC LIMIT 30"
)
try:
aussteller_rows = db.execute(
"SELECT name, halle, stand, land, info, web, themen FROM aussteller LIMIT 1015"
).fetchall()
aussteller_rows = db.execute(sql, params).fetchall()
except Exception:
aussteller_rows = []
if aussteller_rows:
# Score and filter exhibitors by keyword match
# Score and filter exhibitors by keyword match (already SQL-filtered)
scored = []
for a in aussteller_rows:
full_text = f"{a['name'] or ''} {a['halle'] or ''} {a['stand'] or ''} {a['land'] or ''} {a['info'] or ''} {a['themen'] or ''}".lower()
score = sum(1 for t in search_terms if t in full_text)
if score > 0:
score = sum(1 for t in meaningful_terms if t in full_text)
scored.append((score, a))
scored.sort(key=lambda x: -x[0])
# Merge exhibitors with similar names (case-insensitive substring match)
@@ -3129,11 +3161,9 @@ def api_chat():
merge_order = []
for score, a in scored:
name_lower = (a['name'] or '').strip().lower()
# Find existing merge target
found = False
for existing_name in merge_order:
if name_lower in existing_name or existing_name in name_lower:
# Merge stands
existing_stands = merged[existing_name]['stand']
new_stands = (a['stand'] or '').strip()
all_stands = set(s.strip() for s in (existing_stands + '|' + new_stands).split('|') if s.strip())
@@ -3146,10 +3176,8 @@ def api_chat():
merged[name_lower]['stand'] = (a['stand'] or '').strip()
merged[name_lower]['_score'] = score
# Replace scored with merged entries
new_scored = [(merged[n]['_score'], merged[n]) for n in merge_order]
new_scored.sort(key=lambda x: -x[0])
# Add top matching exhibitors
for _, a in new_scored[:20]:
stands_raw = (a['stand'] or '').strip()
stands_list = [s.strip() for s in stands_raw.split('|') if s.strip()]
@@ -3164,24 +3192,34 @@ def api_chat():
+ (f" | {info}" if info else '')
+ (f" | Web: {web}" if web else '')
)
# Also include all exhibitors matching the user's search when they ask about halls/stands
if any(t in user_message.lower() for t in ['halle', 'stand', 'aussteller', 'verlag', 'verlage', 'publisher', 'spiel']):
for _, a in new_scored[:40]:
stands_raw = (a['stand'] or '').strip()
stands_list = [s.strip() for s in stands_raw.split('|') if s.strip()]
halle_stand = ' | '.join(stands_list) if len(stands_list) > 1 else (stands_list[0] if stands_list else '')
land = a['land'] or ''
context_parts.append(
f"[Aussteller] {a['name']}"
+ (f" | {halle_stand}" if halle_stand else '')
+ (f" | {land}" if land else '')
)
# Also search neuheiten table for matching games
# Search neuheiten table using SQL LIKE filters with relevance scoring (OOM-safe)
neuheiten_rows = []
if meaningful_terms:
score_parts_n = []
like_clauses_n = []
params_n = []
for t in meaningful_terms:
term_len = len(t)
score_parts_n.append(
f"(CASE WHEN LOWER(titel) LIKE ? OR LOWER(untertitel) LIKE ? OR LOWER(halle) LIKE ? "
f"OR LOWER(stand) LIKE ? OR LOWER(info_text) LIKE ? OR LOWER(themen) LIKE ? THEN {term_len} ELSE 0 END)"
)
like_clauses_n.append(
"(LOWER(titel) LIKE ? OR LOWER(untertitel) LIKE ? OR LOWER(halle) LIKE ? "
"OR LOWER(stand) LIKE ? OR LOWER(info_text) LIKE ? OR LOWER(themen) LIKE ?)"
)
params_n.extend([f'%{t}%'] * 6)
params_n.extend(params_n[:]) # duplicate for WHERE clause
score_expr_n = ' + '.join(score_parts_n)
sql_n = (
f"SELECT titel, untertitel, halle, stand, info_text, themen, web, ({score_expr_n}) AS relevance "
"FROM neuheiten "
f"WHERE {' OR '.join(like_clauses_n)} "
"ORDER BY relevance DESC LIMIT 30"
)
try:
neuheiten_rows = db.execute(
"SELECT titel, untertitel, halle, stand, info_text, themen, web FROM neuheiten LIMIT 1861"
).fetchall()
neuheiten_rows = db.execute(sql_n, params_n).fetchall()
except Exception:
neuheiten_rows = []
@@ -3189,11 +3227,10 @@ def api_chat():
scored_n = []
for n in neuheiten_rows:
full_text = f"{n['titel'] or ''} {n['untertitel'] or ''} {n['halle'] or ''} {n['stand'] or ''} {n['info_text'] or ''} {n['themen'] or ''}".lower()
score = sum(1 for t in search_terms if t in full_text)
score = sum(1 for t in meaningful_terms if t in full_text)
if score > 0:
scored_n.append((score, n))
scored_n.sort(key=lambda x: -x[0])
# Add top matching games
for _, n in scored_n[:20]:
halle_stand = f"{n['halle'] or ''} {n['stand'] or ''}".strip()
verlag = n['untertitel'] or ''
@@ -3204,16 +3241,6 @@ def api_chat():
+ (f" | {halle_stand}" if halle_stand else '')
+ (f" | {themen}" if themen else '')
)
# For game/neuheit-specific queries, add more results
if any(t in user_message.lower() for t in ['neuheit', 'neuheiten', 'spiel', 'spiele', 'brettspiel', 'kartenspiel', 'würfelspiel', 'game', 'neu']):
for _, n in scored_n[:40]:
halle_stand = f"{n['halle'] or ''} {n['stand'] or ''}".strip()
verlag = n['untertitel'] or ''
context_parts.append(
f"[Neuheit] {n['titel']}"
+ (f" | Verlag: {verlag}" if verlag else '')
+ (f" | {halle_stand}" if halle_stand else '')
)
context = '\n'.join(context_parts)
# System prompt