fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)
This commit is contained in:
Hermes Agent
@@ -0,0 +1,364 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract board game deals from 3 sources, verify on brettspielpreise.de, output JSON."""
|
||||
|
||||
import re
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
import html
|
||||
import time
|
||||
import sys
|
||||
import statistics
|
||||
|
||||
def fetch_brettspielpreise(name):
|
||||
"""Search brettspielpreise.de for a game and return best price + median by scraping HTML."""
|
||||
try:
|
||||
data = urllib.parse.urlencode({"search": name}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
"https://brettspielpreise.de/item/search",
|
||||
data=data,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (compatible; HermesAgent/1.0)",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Accept": "text/html",
|
||||
}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
body = resp.read().decode("utf-8")
|
||||
except Exception as e:
|
||||
print(f" [WARN] HTTP error for '{name}': {e}", file=sys.stderr)
|
||||
return None, None
|
||||
|
||||
prices_all = []
|
||||
|
||||
# Split by result rows (each row starts with itemimg div)
|
||||
result_blocks = re.split(r'<div class="searchcell itemimg', body)
|
||||
|
||||
for block in result_blocks[1:]: # skip header
|
||||
# Only process DE version results
|
||||
if 'alt="DE"' not in block and '/img/svg/flags/DE.svg' not in block:
|
||||
continue
|
||||
|
||||
# Extract all prices from this DE result block
|
||||
# Context: up to 800 chars after the itemimg should contain the prices
|
||||
context = block[:800]
|
||||
price_matches = re.findall(r'<span class="price">€([\d.,]+)</span>', context)
|
||||
for p in price_matches:
|
||||
try:
|
||||
prices_all.append(float(p.replace(",", ".")))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if not prices_all:
|
||||
return None, None
|
||||
|
||||
best = min(prices_all)
|
||||
median = statistics.median(prices_all) if len(prices_all) >= 2 else best
|
||||
return round(best, 2), round(median, 2)
|
||||
|
||||
def parse_amazon(html_content):
|
||||
"""Parse Amazon deals HTML."""
|
||||
deals = []
|
||||
# Split by deal items
|
||||
items = re.split(r'<div style="display:flex;gap:1rem;padding:1rem 0;border-bottom:1px solid #eee;">', html_content)
|
||||
for item in items[1:]: # skip header
|
||||
# Extract name
|
||||
name_m = re.search(r'<strong>(.*?)</strong>', item)
|
||||
if not name_m:
|
||||
continue
|
||||
name = html.unescape(name_m.group(1).strip())
|
||||
|
||||
# Extract price
|
||||
price_m = re.search(r'<span style="color:#b12704;font-size:1\.2rem;font-weight:bold;">([\d.,]+)\s*€</span>', item)
|
||||
if not price_m:
|
||||
continue
|
||||
price = float(price_m.group(1).replace(",", "."))
|
||||
|
||||
# Extract UVP
|
||||
uvp_m = re.search(r'UVP\s*([\d.,]+)\s*€', item)
|
||||
uvp = float(uvp_m.group(1).replace(",", ".")) if uvp_m else price
|
||||
|
||||
# Extract savings percent
|
||||
savings_m = re.search(r'-(\d+)%', item)
|
||||
savings_pct = int(savings_m.group(1)) if savings_m else 0
|
||||
|
||||
# Extract URL
|
||||
url_m = re.search(r'href="(https://www\.amazon\.de/dp/[^"?]+)', item)
|
||||
url = url_m.group(1) if url_m else ""
|
||||
|
||||
deals.append({
|
||||
"name": name,
|
||||
"shop": "Amazon",
|
||||
"price": price,
|
||||
"uvp": uvp,
|
||||
"savings_pct": savings_pct,
|
||||
"url": url
|
||||
})
|
||||
return deals
|
||||
|
||||
def parse_spiele_offensive(html_content):
|
||||
"""Parse Spiele-Offensive deals HTML."""
|
||||
deals = []
|
||||
items = re.split(r'<div class="card">', html_content)
|
||||
for item in items[1:]:
|
||||
# Extract name
|
||||
name_m = re.search(r'<div class="card-name">(.*?)</div>', item)
|
||||
if not name_m:
|
||||
continue
|
||||
name = html.unescape(name_m.group(1).strip())
|
||||
|
||||
# Extract new price
|
||||
new_m = re.search(r'<span class="card-new">([\d.,]+)\s*€</span>', item)
|
||||
if not new_m:
|
||||
continue
|
||||
price = float(new_m.group(1).replace(",", "."))
|
||||
|
||||
# Extract old price
|
||||
old_m = re.search(r'<span class="card-old">([\d.,]+)\s*€</span>', item)
|
||||
uvp = float(old_m.group(1).replace(",", ".")) if old_m else price
|
||||
|
||||
# Extract savings percent
|
||||
pct_m = re.search(r'-(\d+)%', item)
|
||||
savings_pct = int(pct_m.group(1)) if pct_m else 0
|
||||
|
||||
# Extract URL
|
||||
url_m = re.search(r'href="(https://www\.spiele-offensive\.de/[^"]+)"', item)
|
||||
url = url_m.group(1) if url_m else ""
|
||||
|
||||
deals.append({
|
||||
"name": name,
|
||||
"shop": "Spiele-Offensive",
|
||||
"price": price,
|
||||
"uvp": uvp,
|
||||
"savings_pct": savings_pct,
|
||||
"url": url
|
||||
})
|
||||
return deals
|
||||
|
||||
def parse_fantasywelt(html_content):
|
||||
"""Parse Fantasywelt neuheiten HTML - extract Neu im Shop and Wieder auf Lager."""
|
||||
deals = []
|
||||
|
||||
# Parse "Neu im Shop" section
|
||||
neu_section = html_content.split("Neu im Shop")[1].split("Wieder auf Lager")[0] if "Neu im Shop" in html_content and "Wieder auf Lager" in html_content else ""
|
||||
|
||||
# Parse "Wieder auf Lager" section
|
||||
lager_section = html_content.split("Wieder auf Lager")[1] if "Wieder auf Lager" in html_content else ""
|
||||
|
||||
for section in [neu_section, lager_section]:
|
||||
if not section:
|
||||
continue
|
||||
|
||||
# Find all deal blocks
|
||||
blocks = re.findall(r'<a href="(https://www\.fantasywelt\.de/[^"?]+)[^"]*"[^>]*style="[^"]*">(.*?)</a>', section, re.DOTALL)
|
||||
|
||||
# Also extract prices
|
||||
price_blocks = re.findall(
|
||||
r'<a href="(https://www\.fantasywelt\.de/[^"?]+)[^"]*"[^>]*style="[^"]*">(.*?)</a>.*?<span style="color:#(?:e8952e|2563eb);font-size:1\.1rem;font-weight:700;">([\d.,]+)\s*€</span>',
|
||||
section, re.DOTALL
|
||||
)
|
||||
|
||||
for match in price_blocks:
|
||||
url, name_raw, price_str = match
|
||||
name = html.unescape(name_raw.strip())
|
||||
try:
|
||||
price = float(price_str.replace(",", "."))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
deals.append({
|
||||
"name": name,
|
||||
"shop": "Fantasywelt",
|
||||
"price": price,
|
||||
"uvp": price, # No UVP given
|
||||
"savings_pct": 0,
|
||||
"url": url
|
||||
})
|
||||
|
||||
return deals
|
||||
|
||||
def is_board_game(name):
|
||||
"""Filter out non-boardgame items."""
|
||||
name_lower = name.lower()
|
||||
|
||||
# Exclude these categories - pure card decks, accessories, etc.
|
||||
exclude_keywords = [
|
||||
"quartett", "top trumps", "top ass", "schwarzer peter",
|
||||
"rommé", "romme", "bridge canasta", "canasta",
|
||||
"skat", "skatkarten", "spielkarten klassiker",
|
||||
"leiter spiel", "leiterspiel", "metalldose",
|
||||
"deluxe holz", # chess
|
||||
"lernkartenspiel",
|
||||
"krimi-dinner",
|
||||
"hidden games",
|
||||
"mord bei tisch",
|
||||
"miniatures pack", "game mat",
|
||||
"wundertüte", "jubiläumswundertüte",
|
||||
"erweiterung]", "expansion (de)", # pure expansions
|
||||
"logik rätsel", "rätsel", # puzzle books
|
||||
"metalldose", # travel versions in tin boxes
|
||||
]
|
||||
|
||||
for kw in exclude_keywords:
|
||||
if kw in name_lower:
|
||||
return False
|
||||
|
||||
# Known good games that should always be included
|
||||
known_good = [
|
||||
"my city", "catan", "evacuation", "leben in reterra",
|
||||
"das verrückte labyrinth", "funkelschatz", "monopoly",
|
||||
"jumanji", "die verräter", "scrabble",
|
||||
"sagaland", "outsmarted", "die magischen schlüssel",
|
||||
"stille post extrem", "maus reiß aus",
|
||||
"intrepid", "challengers", "next station",
|
||||
"27", "hitster", "the mind",
|
||||
"root", "zug um zug", "dixit",
|
||||
"king of tokyo", "karak", "paleo",
|
||||
"stone age", "robinson crusoe", "sos dino",
|
||||
"t.i.m.e. stories", "pax pamir", "obsession",
|
||||
"marvel remix", "robot quest", "flick of faith",
|
||||
"unmatched", "neuroshima", "fantastic factories",
|
||||
"kitchen rush", "pagan", "wasserkraft",
|
||||
"perdition", "katapult fehde", "fika",
|
||||
"zank am zaun", "onirim", "eleven",
|
||||
"schätz it", "der herr der ringe", "dungeon fighter",
|
||||
"die verdrehte spuknacht", "mythwind", "ein sommernachtstraum",
|
||||
"punkte salat", "4 bilder 1 wort",
|
||||
"grand conquest", "tribe", "hijacked", "influentia",
|
||||
]
|
||||
|
||||
for good in known_good:
|
||||
if good in name_lower:
|
||||
return True
|
||||
|
||||
# General board game indicators
|
||||
include_keywords = [
|
||||
"brettspiel", "gesellschaft", "familien",
|
||||
"strategie", "fantasy", "adventure",
|
||||
"kartenspiel", "würfelspiel",
|
||||
]
|
||||
|
||||
for kw in include_keywords:
|
||||
if kw in name_lower:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def clean_display_name(name):
|
||||
"""Clean a name for display in output JSON."""
|
||||
# Remove HTML entities
|
||||
name = html.unescape(name)
|
||||
name = re.sub(r'\s*\(DE\)\s*', '', name)
|
||||
name = re.sub(r'\s*\((?:EN|Multi|Multilingual)\)\s*', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*\[.*?\]\s*', '', name)
|
||||
# Truncate at comma or dash followed by description words
|
||||
name = re.sub(r'\s*[,–-]\s*(?:Brettspiel|Kartenspiel|Gesellschaftsspiel|Familienspiel|Kinderspiel|Mitbringspiel|Würfelspiel|Aktionsspiel|Strategiespiel|Party.spiel).*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*(?:für|für|ab)\s+\d+.*?(?:Spieler|Personen|Jahren?).*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*-\s*(?:Deutsche (?:Version|Fassung)).*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*-\s*(?:Kennerspiel des Jahres \d+).*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*-\s*(?:Nominiert Spiel des Jahres \d+).*$', '', name, flags=re.IGNORECASE)
|
||||
# Remove trailing punctuation
|
||||
name = re.sub(r'\s*[,;:-]\s*$', '', name)
|
||||
name = re.sub(r'\s+', ' ', name).strip()
|
||||
if len(name) > 100:
|
||||
name = name[:97] + "..."
|
||||
return name
|
||||
|
||||
def clean_name(name):
|
||||
"""Clean up a game name for searching on brettspielpreise.de."""
|
||||
# Remove HTML entities
|
||||
name = html.unescape(name)
|
||||
# Remove parentheticals with extensions/language codes
|
||||
name = re.sub(r'\s*\(DE\)\s*', '', name)
|
||||
name = re.sub(r'\s*\((?:EN|Multi|Multilingual)\)\s*', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*\[.*?\]\s*', '', name)
|
||||
# Remove common suffixes
|
||||
name = re.sub(r'\s*[,–-]\s*(?:Brettspiel|Kartenspiel|Gesellschaftsspiel|Familienspiel|Kinderspiel|Mitbringspiel|Würfelspiel|Aktionsspiel|Strategiespiel|Party.spiel).*$', '', name, flags=re.IGNORECASE)
|
||||
# Remove "für X Spieler" etc
|
||||
name = re.sub(r'\s*(?:für|für|ab)\s+\d+.*?(?:Spieler|Personen|Jahren?).*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*-\s*(?:Deutsche (?:Version|Fassung)|Deutsche Fassung|Deutsche Version).*$', '', name, flags=re.IGNORECASE)
|
||||
# Remove publisher prefixes like "KOSMOS 684655 ", "Schmidt Spiele 49201 ", etc.
|
||||
name = re.sub(r'^(?:KOSMOS|Schmidt Spiele|Ravensburger|Mattel Games|Game Factory|HABA|Hasbro Gaming|Goliath|Spin Master Games|NSV|MATTEL GAMES)\s+\d*\s*', '', name, flags=re.IGNORECASE)
|
||||
# Clean up
|
||||
name = re.sub(r'\s+', ' ', name).strip()
|
||||
name = name[:80].strip()
|
||||
return name
|
||||
|
||||
def main():
|
||||
# Load HTML files
|
||||
with open("/home/hermes/workspace/amazon_deals_2026-06-05.html", "r") as f:
|
||||
amazon_html = f.read()
|
||||
|
||||
with open("/home/hermes/workspace/spiele_offensive_deals.html", "r") as f:
|
||||
so_html = f.read()
|
||||
|
||||
with open("/home/hermes/workspace/fantasywelt_data/neuheiten.html", "r") as f:
|
||||
fw_html = f.read()
|
||||
|
||||
# Parse deals
|
||||
amazon_deals = parse_amazon(amazon_html)
|
||||
so_deals = parse_spiele_offensive(so_html)
|
||||
fw_deals = parse_fantasywelt(fw_html)
|
||||
|
||||
print(f"Extracted: {len(amazon_deals)} Amazon, {len(so_deals)} Spiele-Offensive, {len(fw_deals)} Fantasywelt")
|
||||
|
||||
# Combine and filter
|
||||
all_deals = amazon_deals + so_deals + fw_deals
|
||||
|
||||
# Filter for board games
|
||||
filtered = [d for d in all_deals if is_board_game(d["name"])]
|
||||
print(f"After filtering: {len(filtered)} deals")
|
||||
|
||||
# Sort by savings_pct descending, take top 30 to check
|
||||
filtered.sort(key=lambda x: x["savings_pct"], reverse=True)
|
||||
top_candidates = filtered[:30]
|
||||
|
||||
# Verify prices on brettspielpreise.de
|
||||
verified = []
|
||||
for i, deal in enumerate(top_candidates):
|
||||
search_name = clean_name(deal["name"])
|
||||
print(f"[{i+1}/{len(top_candidates)}] Checking: {search_name}")
|
||||
|
||||
best_other, median_market = fetch_brettspielpreise(search_name)
|
||||
|
||||
# Determine verdict
|
||||
if best_other is not None:
|
||||
if deal["price"] <= best_other:
|
||||
verdict = "gut"
|
||||
elif deal["price"] <= best_other * 1.15:
|
||||
verdict = "durchschnittlich"
|
||||
else:
|
||||
verdict = "schlecht"
|
||||
else:
|
||||
verdict = "unbekannt"
|
||||
|
||||
verified.append({
|
||||
"name": clean_display_name(deal["name"]),
|
||||
"shop": deal["shop"],
|
||||
"price": deal["price"],
|
||||
"uvp": deal["uvp"],
|
||||
"savings_pct": deal["savings_pct"],
|
||||
"url": deal["url"],
|
||||
"best_other_price": best_other,
|
||||
"median_market_price": median_market,
|
||||
"verdict": verdict
|
||||
})
|
||||
|
||||
# Rate limit
|
||||
time.sleep(1)
|
||||
|
||||
# Sort final by savings_pct, take top 12
|
||||
verified.sort(key=lambda x: x["savings_pct"], reverse=True)
|
||||
top12 = verified[:12]
|
||||
|
||||
# Write JSON
|
||||
with open("/tmp/deals_verified.json", "w") as f:
|
||||
json.dump(top12, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nDone! Wrote {len(top12)} deals to /tmp/deals_verified.json")
|
||||
for d in top12:
|
||||
print(f" {d['savings_pct']:3d}% | {d['shop']:20s} | {d['price']:6.2f}€ | {d['name'][:60]}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user