Hermes Agent
365 lines
14 KiB
Python
365 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract board game deals from 3 sources, verify on brettspielpreise.de, output JSON."""
|
||
|
||
import re
|
||
import json
|
||
import urllib.request
|
||
import urllib.parse
|
||
import urllib.error
|
||
import html
|
||
import time
|
||
import sys
|
||
import statistics
|
||
|
||
def fetch_brettspielpreise(name):
|
||
"""Search brettspielpreise.de for a game and return best price + median by scraping HTML."""
|
||
try:
|
||
data = urllib.parse.urlencode({"search": name}).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
"https://brettspielpreise.de/item/search",
|
||
data=data,
|
||
headers={
|
||
"User-Agent": "Mozilla/5.0 (compatible; HermesAgent/1.0)",
|
||
"Content-Type": "application/x-www-form-urlencoded",
|
||
"Accept": "text/html",
|
||
}
|
||
)
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
body = resp.read().decode("utf-8")
|
||
except Exception as e:
|
||
print(f" [WARN] HTTP error for '{name}': {e}", file=sys.stderr)
|
||
return None, None
|
||
|
||
prices_all = []
|
||
|
||
# Split by result rows (each row starts with itemimg div)
|
||
result_blocks = re.split(r'<div class="searchcell itemimg', body)
|
||
|
||
for block in result_blocks[1:]: # skip header
|
||
# Only process DE version results
|
||
if 'alt="DE"' not in block and '/img/svg/flags/DE.svg' not in block:
|
||
continue
|
||
|
||
# Extract all prices from this DE result block
|
||
# Context: up to 800 chars after the itemimg should contain the prices
|
||
context = block[:800]
|
||
price_matches = re.findall(r'<span class="price">€([\d.,]+)</span>', context)
|
||
for p in price_matches:
|
||
try:
|
||
prices_all.append(float(p.replace(",", ".")))
|
||
except ValueError:
|
||
pass
|
||
|
||
if not prices_all:
|
||
return None, None
|
||
|
||
best = min(prices_all)
|
||
median = statistics.median(prices_all) if len(prices_all) >= 2 else best
|
||
return round(best, 2), round(median, 2)
|
||
|
||
def parse_amazon(html_content):
|
||
"""Parse Amazon deals HTML."""
|
||
deals = []
|
||
# Split by deal items
|
||
items = re.split(r'<div style="display:flex;gap:1rem;padding:1rem 0;border-bottom:1px solid #eee;">', html_content)
|
||
for item in items[1:]: # skip header
|
||
# Extract name
|
||
name_m = re.search(r'<strong>(.*?)</strong>', item)
|
||
if not name_m:
|
||
continue
|
||
name = html.unescape(name_m.group(1).strip())
|
||
|
||
# Extract price
|
||
price_m = re.search(r'<span style="color:#b12704;font-size:1\.2rem;font-weight:bold;">([\d.,]+)\s*€</span>', item)
|
||
if not price_m:
|
||
continue
|
||
price = float(price_m.group(1).replace(",", "."))
|
||
|
||
# Extract UVP
|
||
uvp_m = re.search(r'UVP\s*([\d.,]+)\s*€', item)
|
||
uvp = float(uvp_m.group(1).replace(",", ".")) if uvp_m else price
|
||
|
||
# Extract savings percent
|
||
savings_m = re.search(r'-(\d+)%', item)
|
||
savings_pct = int(savings_m.group(1)) if savings_m else 0
|
||
|
||
# Extract URL
|
||
url_m = re.search(r'href="(https://www\.amazon\.de/dp/[^"?]+)', item)
|
||
url = url_m.group(1) if url_m else ""
|
||
|
||
deals.append({
|
||
"name": name,
|
||
"shop": "Amazon",
|
||
"price": price,
|
||
"uvp": uvp,
|
||
"savings_pct": savings_pct,
|
||
"url": url
|
||
})
|
||
return deals
|
||
|
||
def parse_spiele_offensive(html_content):
|
||
"""Parse Spiele-Offensive deals HTML."""
|
||
deals = []
|
||
items = re.split(r'<div class="card">', html_content)
|
||
for item in items[1:]:
|
||
# Extract name
|
||
name_m = re.search(r'<div class="card-name">(.*?)</div>', item)
|
||
if not name_m:
|
||
continue
|
||
name = html.unescape(name_m.group(1).strip())
|
||
|
||
# Extract new price
|
||
new_m = re.search(r'<span class="card-new">([\d.,]+)\s*€</span>', item)
|
||
if not new_m:
|
||
continue
|
||
price = float(new_m.group(1).replace(",", "."))
|
||
|
||
# Extract old price
|
||
old_m = re.search(r'<span class="card-old">([\d.,]+)\s*€</span>', item)
|
||
uvp = float(old_m.group(1).replace(",", ".")) if old_m else price
|
||
|
||
# Extract savings percent
|
||
pct_m = re.search(r'-(\d+)%', item)
|
||
savings_pct = int(pct_m.group(1)) if pct_m else 0
|
||
|
||
# Extract URL
|
||
url_m = re.search(r'href="(https://www\.spiele-offensive\.de/[^"]+)"', item)
|
||
url = url_m.group(1) if url_m else ""
|
||
|
||
deals.append({
|
||
"name": name,
|
||
"shop": "Spiele-Offensive",
|
||
"price": price,
|
||
"uvp": uvp,
|
||
"savings_pct": savings_pct,
|
||
"url": url
|
||
})
|
||
return deals
|
||
|
||
def parse_fantasywelt(html_content):
|
||
"""Parse Fantasywelt neuheiten HTML - extract Neu im Shop and Wieder auf Lager."""
|
||
deals = []
|
||
|
||
# Parse "Neu im Shop" section
|
||
neu_section = html_content.split("Neu im Shop")[1].split("Wieder auf Lager")[0] if "Neu im Shop" in html_content and "Wieder auf Lager" in html_content else ""
|
||
|
||
# Parse "Wieder auf Lager" section
|
||
lager_section = html_content.split("Wieder auf Lager")[1] if "Wieder auf Lager" in html_content else ""
|
||
|
||
for section in [neu_section, lager_section]:
|
||
if not section:
|
||
continue
|
||
|
||
# Find all deal blocks
|
||
blocks = re.findall(r'<a href="(https://www\.fantasywelt\.de/[^"?]+)[^"]*"[^>]*style="[^"]*">(.*?)</a>', section, re.DOTALL)
|
||
|
||
# Also extract prices
|
||
price_blocks = re.findall(
|
||
r'<a href="(https://www\.fantasywelt\.de/[^"?]+)[^"]*"[^>]*style="[^"]*">(.*?)</a>.*?<span style="color:#(?:e8952e|2563eb);font-size:1\.1rem;font-weight:700;">([\d.,]+)\s*€</span>',
|
||
section, re.DOTALL
|
||
)
|
||
|
||
for match in price_blocks:
|
||
url, name_raw, price_str = match
|
||
name = html.unescape(name_raw.strip())
|
||
try:
|
||
price = float(price_str.replace(",", "."))
|
||
except ValueError:
|
||
continue
|
||
|
||
deals.append({
|
||
"name": name,
|
||
"shop": "Fantasywelt",
|
||
"price": price,
|
||
"uvp": price, # No UVP given
|
||
"savings_pct": 0,
|
||
"url": url
|
||
})
|
||
|
||
return deals
|
||
|
||
def is_board_game(name):
|
||
"""Filter out non-boardgame items."""
|
||
name_lower = name.lower()
|
||
|
||
# Exclude these categories - pure card decks, accessories, etc.
|
||
exclude_keywords = [
|
||
"quartett", "top trumps", "top ass", "schwarzer peter",
|
||
"rommé", "romme", "bridge canasta", "canasta",
|
||
"skat", "skatkarten", "spielkarten klassiker",
|
||
"leiter spiel", "leiterspiel", "metalldose",
|
||
"deluxe holz", # chess
|
||
"lernkartenspiel",
|
||
"krimi-dinner",
|
||
"hidden games",
|
||
"mord bei tisch",
|
||
"miniatures pack", "game mat",
|
||
"wundertüte", "jubiläumswundertüte",
|
||
"erweiterung]", "expansion (de)", # pure expansions
|
||
"logik rätsel", "rätsel", # puzzle books
|
||
"metalldose", # travel versions in tin boxes
|
||
]
|
||
|
||
for kw in exclude_keywords:
|
||
if kw in name_lower:
|
||
return False
|
||
|
||
# Known good games that should always be included
|
||
known_good = [
|
||
"my city", "catan", "evacuation", "leben in reterra",
|
||
"das verrückte labyrinth", "funkelschatz", "monopoly",
|
||
"jumanji", "die verräter", "scrabble",
|
||
"sagaland", "outsmarted", "die magischen schlüssel",
|
||
"stille post extrem", "maus reiß aus",
|
||
"intrepid", "challengers", "next station",
|
||
"27", "hitster", "the mind",
|
||
"root", "zug um zug", "dixit",
|
||
"king of tokyo", "karak", "paleo",
|
||
"stone age", "robinson crusoe", "sos dino",
|
||
"t.i.m.e. stories", "pax pamir", "obsession",
|
||
"marvel remix", "robot quest", "flick of faith",
|
||
"unmatched", "neuroshima", "fantastic factories",
|
||
"kitchen rush", "pagan", "wasserkraft",
|
||
"perdition", "katapult fehde", "fika",
|
||
"zank am zaun", "onirim", "eleven",
|
||
"schätz it", "der herr der ringe", "dungeon fighter",
|
||
"die verdrehte spuknacht", "mythwind", "ein sommernachtstraum",
|
||
"punkte salat", "4 bilder 1 wort",
|
||
"grand conquest", "tribe", "hijacked", "influentia",
|
||
]
|
||
|
||
for good in known_good:
|
||
if good in name_lower:
|
||
return True
|
||
|
||
# General board game indicators
|
||
include_keywords = [
|
||
"brettspiel", "gesellschaft", "familien",
|
||
"strategie", "fantasy", "adventure",
|
||
"kartenspiel", "würfelspiel",
|
||
]
|
||
|
||
for kw in include_keywords:
|
||
if kw in name_lower:
|
||
return True
|
||
|
||
return False
|
||
|
||
def clean_display_name(name):
|
||
"""Clean a name for display in output JSON."""
|
||
# Remove HTML entities
|
||
name = html.unescape(name)
|
||
name = re.sub(r'\s*\(DE\)\s*', '', name)
|
||
name = re.sub(r'\s*\((?:EN|Multi|Multilingual)\)\s*', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*\[.*?\]\s*', '', name)
|
||
# Truncate at comma or dash followed by description words
|
||
name = re.sub(r'\s*[,–-]\s*(?:Brettspiel|Kartenspiel|Gesellschaftsspiel|Familienspiel|Kinderspiel|Mitbringspiel|Würfelspiel|Aktionsspiel|Strategiespiel|Party.spiel).*$', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*(?:für|für|ab)\s+\d+.*?(?:Spieler|Personen|Jahren?).*$', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*-\s*(?:Deutsche (?:Version|Fassung)).*$', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*-\s*(?:Kennerspiel des Jahres \d+).*$', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*-\s*(?:Nominiert Spiel des Jahres \d+).*$', '', name, flags=re.IGNORECASE)
|
||
# Remove trailing punctuation
|
||
name = re.sub(r'\s*[,;:-]\s*$', '', name)
|
||
name = re.sub(r'\s+', ' ', name).strip()
|
||
if len(name) > 100:
|
||
name = name[:97] + "..."
|
||
return name
|
||
|
||
def clean_name(name):
|
||
"""Clean up a game name for searching on brettspielpreise.de."""
|
||
# Remove HTML entities
|
||
name = html.unescape(name)
|
||
# Remove parentheticals with extensions/language codes
|
||
name = re.sub(r'\s*\(DE\)\s*', '', name)
|
||
name = re.sub(r'\s*\((?:EN|Multi|Multilingual)\)\s*', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*\[.*?\]\s*', '', name)
|
||
# Remove common suffixes
|
||
name = re.sub(r'\s*[,–-]\s*(?:Brettspiel|Kartenspiel|Gesellschaftsspiel|Familienspiel|Kinderspiel|Mitbringspiel|Würfelspiel|Aktionsspiel|Strategiespiel|Party.spiel).*$', '', name, flags=re.IGNORECASE)
|
||
# Remove "für X Spieler" etc
|
||
name = re.sub(r'\s*(?:für|für|ab)\s+\d+.*?(?:Spieler|Personen|Jahren?).*$', '', name, flags=re.IGNORECASE)
|
||
name = re.sub(r'\s*-\s*(?:Deutsche (?:Version|Fassung)|Deutsche Fassung|Deutsche Version).*$', '', name, flags=re.IGNORECASE)
|
||
# Remove publisher prefixes like "KOSMOS 684655 ", "Schmidt Spiele 49201 ", etc.
|
||
name = re.sub(r'^(?:KOSMOS|Schmidt Spiele|Ravensburger|Mattel Games|Game Factory|HABA|Hasbro Gaming|Goliath|Spin Master Games|NSV|MATTEL GAMES)\s+\d*\s*', '', name, flags=re.IGNORECASE)
|
||
# Clean up
|
||
name = re.sub(r'\s+', ' ', name).strip()
|
||
name = name[:80].strip()
|
||
return name
|
||
|
||
def main():
|
||
# Load HTML files
|
||
with open("/home/hermes/workspace/amazon_deals_2026-06-05.html", "r") as f:
|
||
amazon_html = f.read()
|
||
|
||
with open("/home/hermes/workspace/spiele_offensive_deals.html", "r") as f:
|
||
so_html = f.read()
|
||
|
||
with open("/home/hermes/workspace/fantasywelt_data/neuheiten.html", "r") as f:
|
||
fw_html = f.read()
|
||
|
||
# Parse deals
|
||
amazon_deals = parse_amazon(amazon_html)
|
||
so_deals = parse_spiele_offensive(so_html)
|
||
fw_deals = parse_fantasywelt(fw_html)
|
||
|
||
print(f"Extracted: {len(amazon_deals)} Amazon, {len(so_deals)} Spiele-Offensive, {len(fw_deals)} Fantasywelt")
|
||
|
||
# Combine and filter
|
||
all_deals = amazon_deals + so_deals + fw_deals
|
||
|
||
# Filter for board games
|
||
filtered = [d for d in all_deals if is_board_game(d["name"])]
|
||
print(f"After filtering: {len(filtered)} deals")
|
||
|
||
# Sort by savings_pct descending, take top 30 to check
|
||
filtered.sort(key=lambda x: x["savings_pct"], reverse=True)
|
||
top_candidates = filtered[:30]
|
||
|
||
# Verify prices on brettspielpreise.de
|
||
verified = []
|
||
for i, deal in enumerate(top_candidates):
|
||
search_name = clean_name(deal["name"])
|
||
print(f"[{i+1}/{len(top_candidates)}] Checking: {search_name}")
|
||
|
||
best_other, median_market = fetch_brettspielpreise(search_name)
|
||
|
||
# Determine verdict
|
||
if best_other is not None:
|
||
if deal["price"] <= best_other:
|
||
verdict = "gut"
|
||
elif deal["price"] <= best_other * 1.15:
|
||
verdict = "durchschnittlich"
|
||
else:
|
||
verdict = "schlecht"
|
||
else:
|
||
verdict = "unbekannt"
|
||
|
||
verified.append({
|
||
"name": clean_display_name(deal["name"]),
|
||
"shop": deal["shop"],
|
||
"price": deal["price"],
|
||
"uvp": deal["uvp"],
|
||
"savings_pct": deal["savings_pct"],
|
||
"url": deal["url"],
|
||
"best_other_price": best_other,
|
||
"median_market_price": median_market,
|
||
"verdict": verdict
|
||
})
|
||
|
||
# Rate limit
|
||
time.sleep(1)
|
||
|
||
# Sort final by savings_pct, take top 12
|
||
verified.sort(key=lambda x: x["savings_pct"], reverse=True)
|
||
top12 = verified[:12]
|
||
|
||
# Write JSON
|
||
with open("/tmp/deals_verified.json", "w") as f:
|
||
json.dump(top12, f, indent=2, ensure_ascii=False)
|
||
|
||
print(f"\nDone! Wrote {len(top12)} deals to /tmp/deals_verified.json")
|
||
for d in top12:
|
||
print(f" {d['savings_pct']:3d}% | {d['shop']:20s} | {d['price']:6.2f}€ | {d['name'][:60]}")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|