BSN-Chatsystem/parse_neuheiten_kw28.py

#!/usr/bin/env python3
"""Parse neuheiten.html and extract game data for KW28 article."""

import re
import html as html_mod
import json

with open('/home/hermes/workspace/fantasywelt_data/neuheiten.html', 'r') as f:
    content = f.read()

# Split into sections
# Section 1: Neu im Shop (before "Wieder auf Lager")
neu_section = content.split('Wieder auf Lager')[0]

# Section 2: Wieder auf Lager (between "Wieder auf Lager" and "Neue Vorbestellungen")
rest = content.split('Wieder auf Lager')[1] if 'Wieder auf Lager' in content else ''
wieder_section = rest.split('Neue Vorbestellungen')[0] if 'Neue Vorbestellungen' in rest else rest

# Section 3: Vorbestellungen
vorbestell_section = content.split('Neue Vorbestellungen')[1] if 'Neue Vorbestellungen' in content else ''

# Non-game filter keywords (normalized: lowercase, hyphens replaced with spaces)
EXCLUDE_KEYWORDS = [
    # Puzzles
    'puzzle',
    # Sleeves / card protection
    'sleeve', 'deck box', 'deckbox', 'boulder', 'portfolio',
    'ultimate guard', 'sidewinder', 'omnihive', 'xenoskin',
    'card binder', 'deck protector', 'toploader',
    'perfect fit', 'inner sleeve', 'outer sleeve', 'matte sleeve',
    'sammelalbum', 'playmat tube',
    # Playmats
    'spielmatte', 'playmat', 'game mat', 'neoprene', 'play mat',
    # TCG Displays with ORDERUNIT
    'orderunit',
    # Upgrade kits / inserts / organizers
    'upgrade pack', 'upgrade kit', 'insert', 'organizer', 'organiser',
    # Non-games
    'ausgrabung', 'kristall', 'farblabor',
    'dice set', 'wuerfelset', 'wuerfel set', 'dice pack', 'token set',
    'miniatures pack', 'playerboard',
    # Storage / accessories
    'storage box', 'token box', 'token tray', 'bit box',
    'feldherr', 'schaumstoff', 'plastic tray',
    'deck box', 'deckbox',
    'spiel knobelblock', 'knobelblock',
    'anatomiepuzzle',
    # TCG spotlight decks (these are accessories, not games)
    'spotlight deck', 'spotlight-deck',
    # Non-game items
    'gaming tokens', 'gamegenic', 'alcove flip box',
    'play mat', 'premium game mat',
    # Minifiguren blind bags (not games)
    'minifiguren', 'blind bag',
    # Decals / propellers (Dockfighters accessories)
    'decals set', 'decal set', 'propeller set',
    'leather folder',
    # Maps (not games)
    'global map',
    # Tarot decks (not board games)
    'tarot deck',
    # Organizer
    'organizer',
    # Würfel blind box
    'würfel blind box', 'wuerfel blind box',
    # Deluxe upgrade (not a game)
    'deluxe upgrade',
    # Abdeckung
    'abdeckung',
    # Note: "Exclusive Extras" for Dune etc. are legitimate game expansions, NOT excluded
]

def is_excluded(name):
    """Check if a product name matches any exclusion keyword."""
    name_lower = name.lower().replace('-', ' ').replace('  ', ' ')
    for kw in EXCLUDE_KEYWORDS:
        if kw in name_lower:
            return True
    return False

def parse_cards(section):
    """Extract (name, price, url, language, ersch_date, ausverkauft) from HTML section."""
    cards = re.split(
        r'<div style="display:flex;gap:0\.8rem;padding:1rem 0;'
        r'border-bottom:1px solid #eee;align-items:flex-start;">',
        section
    )
    cards = cards[1:]  # first split is header
    
    parsed = []
    for card in cards:
        # Name from first link
        name_match = re.search(
            r'<a href="(https://www\.fantasywelt\.de/[^"]+)"[^>]*>([^<]+)</a>',
            card
        )
        if not name_match:
            continue
        url = name_match.group(1)
        name = html_mod.unescape(name_match.group(2).strip())
        
        # Skip "Kaufen →" links
        if 'Kaufen' in name or '\u2192' in name or len(name) < 3:
            continue
        
        # Apply exclusion filter
        if is_excluded(name):
            continue
        
        # Language badge
        lang_match = re.search(r'background:#e8f5e9;color:#16a34a;[^>]*>([A-Z]+)</span>', card)
        language = lang_match.group(1) if lang_match else ''
        
        # Ersch. date
        ersch_match = re.search(r'Ersch\.\s*(\d{2}\.\d{2}\.\d{4})', card)
        ersch_date = ersch_match.group(1) if ersch_match else ''
        
        # Ausverkauft
        ausverkauft = 'Ausverkauft' in card
        
        # Price
        price_match = re.search(
            r'color:#20228a;font-size:1\.1rem;font-weight:700;">([^<]+)</span>',
            card
        )
        price_raw = price_match.group(1).strip() if price_match else '0'
        # "49,49 €" -> 49.49
        price_str = price_raw.replace(' €', '').replace('€', '').replace('.', '').replace(',', '.').strip()
        try:
            price = float(price_str)
        except:
            price = 0.0
        
        parsed.append({
            'name': name,
            'price': price,
            'price_display': price_raw,
            'url': url,
            'language': language,
            'ersch_date': ersch_date,
            'ausverkauft': ausverkauft,
        })
    
    return parsed

# Parse all sections
neuheiten = parse_cards(neu_section)
wieder_da = parse_cards(wieder_section)
vorbestellungen = parse_cards(vorbestell_section)

# Print stats
print(f"=== PARSE RESULTS ===")
print(f"Neu im Shop: {len(neuheiten)} (after filtering)")
print(f"Wieder auf Lager: {len(wieder_da)} (after filtering)")
print(f"Vorbestellungen: {len(vorbestellungen)} (after filtering)")
print()

# Print first 5 of each for verification
print("=== NEU IM SHOP (first 5) ===")
for item in neuheiten[:5]:
    status = "AUSVERKAUFT" if item['ausverkauft'] else "verfügbar"
    print(f"  {item['name']} | {item['price_display']} | {item['language']} | {item['ersch_date']} | {status}")

print()
print("=== WIEDER AUF LAGER (first 5) ===")
for item in wieder_da[:5]:
    status = "AUSVERKAUFT" if item['ausverkauft'] else "verfügbar"
    print(f"  {item['name']} | {item['price_display']} | {item['language']} | {item['ersch_date']} | {status}")

print()
print("=== VORBESTELLUNGEN (all) ===")
for item in vorbestellungen:
    status = "AUSVERKAUFT" if item['ausverkauft'] else "verfügbar"
    print(f"  {item['name']} | {item['price_display']} | {item['language']} | {item['ersch_date']} | {status}")

# Save parsed data as JSON for article generation
output = {
    'neuheiten': neuheiten,
    'wieder_da': wieder_da,
    'vorbestellungen': vorbestellungen,
}
with open('/home/hermes/workspace/fantasywelt_data/kw28_parsed.json', 'w') as f:
    json.dump(output, f, ensure_ascii=False, indent=2)

print(f"\nSaved {len(neuheiten)} + {len(wieder_da)} + {len(vorbestellungen)} items to kw28_parsed.json")