fix: <style>-Blöcke aus allen 24 Artikeln entfernt + Skill korrigiert
- artikel_*.html: 24 Dateien von <style>-Blöcken befreit (body, max-width zerstören Joomla) - journalistic-article-writing SKILL.md: falsche 'AUSNAHME 09.07.' entfernt - Ursache: Skill-Kommentar erlaubte Sub-Agenten <style> 'für Vorschau' — kein Gatekeeper existiert
This commit is contained in:
Hermes Agent
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse neuheiten.html and extract game data for KW28 article."""
|
||||
|
||||
import re
|
||||
import html as html_mod
|
||||
import json
|
||||
|
||||
with open('/home/hermes/workspace/fantasywelt_data/neuheiten.html', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Split into sections
|
||||
# Section 1: Neu im Shop (before "Wieder auf Lager")
|
||||
neu_section = content.split('Wieder auf Lager')[0]
|
||||
|
||||
# Section 2: Wieder auf Lager (between "Wieder auf Lager" and "Neue Vorbestellungen")
|
||||
rest = content.split('Wieder auf Lager')[1] if 'Wieder auf Lager' in content else ''
|
||||
wieder_section = rest.split('Neue Vorbestellungen')[0] if 'Neue Vorbestellungen' in rest else rest
|
||||
|
||||
# Section 3: Vorbestellungen
|
||||
vorbestell_section = content.split('Neue Vorbestellungen')[1] if 'Neue Vorbestellungen' in content else ''
|
||||
|
||||
# Non-game filter keywords (normalized: lowercase, hyphens replaced with spaces)
|
||||
EXCLUDE_KEYWORDS = [
|
||||
# Puzzles
|
||||
'puzzle',
|
||||
# Sleeves / card protection
|
||||
'sleeve', 'deck box', 'deckbox', 'boulder', 'portfolio',
|
||||
'ultimate guard', 'sidewinder', 'omnihive', 'xenoskin',
|
||||
'card binder', 'deck protector', 'toploader',
|
||||
'perfect fit', 'inner sleeve', 'outer sleeve', 'matte sleeve',
|
||||
'sammelalbum', 'playmat tube',
|
||||
# Playmats
|
||||
'spielmatte', 'playmat', 'game mat', 'neoprene', 'play mat',
|
||||
# TCG Displays with ORDERUNIT
|
||||
'orderunit',
|
||||
# Upgrade kits / inserts / organizers
|
||||
'upgrade pack', 'upgrade kit', 'insert', 'organizer', 'organiser',
|
||||
# Non-games
|
||||
'ausgrabung', 'kristall', 'farblabor',
|
||||
'dice set', 'wuerfelset', 'wuerfel set', 'dice pack', 'token set',
|
||||
'miniatures pack', 'playerboard',
|
||||
# Storage / accessories
|
||||
'storage box', 'token box', 'token tray', 'bit box',
|
||||
'feldherr', 'schaumstoff', 'plastic tray',
|
||||
'deck box', 'deckbox',
|
||||
'spiel knobelblock', 'knobelblock',
|
||||
'anatomiepuzzle',
|
||||
# TCG spotlight decks (these are accessories, not games)
|
||||
'spotlight deck', 'spotlight-deck',
|
||||
# Non-game items
|
||||
'gaming tokens', 'gamegenic', 'alcove flip box',
|
||||
'play mat', 'premium game mat',
|
||||
# Minifiguren blind bags (not games)
|
||||
'minifiguren', 'blind bag',
|
||||
# Decals / propellers (Dockfighters accessories)
|
||||
'decals set', 'decal set', 'propeller set',
|
||||
'leather folder',
|
||||
# Maps (not games)
|
||||
'global map',
|
||||
# Tarot decks (not board games)
|
||||
'tarot deck',
|
||||
# Organizer
|
||||
'organizer',
|
||||
# Würfel blind box
|
||||
'würfel blind box', 'wuerfel blind box',
|
||||
# Deluxe upgrade (not a game)
|
||||
'deluxe upgrade',
|
||||
# Abdeckung
|
||||
'abdeckung',
|
||||
# Note: "Exclusive Extras" for Dune etc. are legitimate game expansions, NOT excluded
|
||||
]
|
||||
|
||||
def is_excluded(name):
|
||||
"""Check if a product name matches any exclusion keyword."""
|
||||
name_lower = name.lower().replace('-', ' ').replace(' ', ' ')
|
||||
for kw in EXCLUDE_KEYWORDS:
|
||||
if kw in name_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_cards(section):
|
||||
"""Extract (name, price, url, language, ersch_date, ausverkauft) from HTML section."""
|
||||
cards = re.split(
|
||||
r'<div style="display:flex;gap:0\.8rem;padding:1rem 0;'
|
||||
r'border-bottom:1px solid #eee;align-items:flex-start;">',
|
||||
section
|
||||
)
|
||||
cards = cards[1:] # first split is header
|
||||
|
||||
parsed = []
|
||||
for card in cards:
|
||||
# Name from first link
|
||||
name_match = re.search(
|
||||
r'<a href="(https://www\.fantasywelt\.de/[^"]+)"[^>]*>([^<]+)</a>',
|
||||
card
|
||||
)
|
||||
if not name_match:
|
||||
continue
|
||||
url = name_match.group(1)
|
||||
name = html_mod.unescape(name_match.group(2).strip())
|
||||
|
||||
# Skip "Kaufen →" links
|
||||
if 'Kaufen' in name or '\u2192' in name or len(name) < 3:
|
||||
continue
|
||||
|
||||
# Apply exclusion filter
|
||||
if is_excluded(name):
|
||||
continue
|
||||
|
||||
# Language badge
|
||||
lang_match = re.search(r'background:#e8f5e9;color:#16a34a;[^>]*>([A-Z]+)</span>', card)
|
||||
language = lang_match.group(1) if lang_match else ''
|
||||
|
||||
# Ersch. date
|
||||
ersch_match = re.search(r'Ersch\.\s*(\d{2}\.\d{2}\.\d{4})', card)
|
||||
ersch_date = ersch_match.group(1) if ersch_match else ''
|
||||
|
||||
# Ausverkauft
|
||||
ausverkauft = 'Ausverkauft' in card
|
||||
|
||||
# Price
|
||||
price_match = re.search(
|
||||
r'color:#20228a;font-size:1\.1rem;font-weight:700;">([^<]+)</span>',
|
||||
card
|
||||
)
|
||||
price_raw = price_match.group(1).strip() if price_match else '0'
|
||||
# "49,49 €" -> 49.49
|
||||
price_str = price_raw.replace(' €', '').replace('€', '').replace('.', '').replace(',', '.').strip()
|
||||
try:
|
||||
price = float(price_str)
|
||||
except:
|
||||
price = 0.0
|
||||
|
||||
parsed.append({
|
||||
'name': name,
|
||||
'price': price,
|
||||
'price_display': price_raw,
|
||||
'url': url,
|
||||
'language': language,
|
||||
'ersch_date': ersch_date,
|
||||
'ausverkauft': ausverkauft,
|
||||
})
|
||||
|
||||
return parsed
|
||||
|
||||
# Parse all sections
|
||||
neuheiten = parse_cards(neu_section)
|
||||
wieder_da = parse_cards(wieder_section)
|
||||
vorbestellungen = parse_cards(vorbestell_section)
|
||||
|
||||
# Print stats
|
||||
print(f"=== PARSE RESULTS ===")
|
||||
print(f"Neu im Shop: {len(neuheiten)} (after filtering)")
|
||||
print(f"Wieder auf Lager: {len(wieder_da)} (after filtering)")
|
||||
print(f"Vorbestellungen: {len(vorbestellungen)} (after filtering)")
|
||||
print()
|
||||
|
||||
# Print first 5 of each for verification
|
||||
print("=== NEU IM SHOP (first 5) ===")
|
||||
for item in neuheiten[:5]:
|
||||
status = "AUSVERKAUFT" if item['ausverkauft'] else "verfügbar"
|
||||
print(f" {item['name']} | {item['price_display']} | {item['language']} | {item['ersch_date']} | {status}")
|
||||
|
||||
print()
|
||||
print("=== WIEDER AUF LAGER (first 5) ===")
|
||||
for item in wieder_da[:5]:
|
||||
status = "AUSVERKAUFT" if item['ausverkauft'] else "verfügbar"
|
||||
print(f" {item['name']} | {item['price_display']} | {item['language']} | {item['ersch_date']} | {status}")
|
||||
|
||||
print()
|
||||
print("=== VORBESTELLUNGEN (all) ===")
|
||||
for item in vorbestellungen:
|
||||
status = "AUSVERKAUFT" if item['ausverkauft'] else "verfügbar"
|
||||
print(f" {item['name']} | {item['price_display']} | {item['language']} | {item['ersch_date']} | {status}")
|
||||
|
||||
# Save parsed data as JSON for article generation
|
||||
output = {
|
||||
'neuheiten': neuheiten,
|
||||
'wieder_da': wieder_da,
|
||||
'vorbestellungen': vorbestellungen,
|
||||
}
|
||||
with open('/home/hermes/workspace/fantasywelt_data/kw28_parsed.json', 'w') as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"\nSaved {len(neuheiten)} + {len(wieder_da)} + {len(vorbestellungen)} items to kw28_parsed.json")
|
||||
Reference in New Issue
Block a user