#!/usr/bin/env python3 """Scrape BGA game panels for detail extraction.""" import subprocess import re import json import time AGENT_BROWSER = "/home/hermes/.hermes/node/lib/node_modules/agent-browser/bin/agent-browser-linux-x64" def run(cmd, timeout=15): result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) return result.stdout # Game panels to scrape panels = [ ("perch", 1101), ("solarsentinels", 1103), ("tanglewoodsred", 1103), ] for slug, news_id in panels: run(f'{AGENT_BROWSER} close --all 2>/dev/null', timeout=3) print(f"\n=== Game Panel: {slug} (news ID {news_id}) ===") open_out = run(f'{AGENT_BROWSER} open "https://boardgamearena.com/gamepanel?game={slug}" --args "--no-sandbox"', timeout=20) time.sleep(2) snap = run(f'{AGENT_BROWSER} snapshot', timeout=15) # Save full snapshot with open(f'/home/hermes/workspace/bga_panel_{slug}.txt', 'w') as f: f.write(snap) # Extract structured data # Designer designers = re.findall(r'StaticText\s+\"(?:A game by|Designer|Designers?):?\s*\"?\s*(.*?)\"', snap, re.IGNORECASE) # Publisher publishers = re.findall(r'StaticText\s+\"(?:Published by|Publisher):?\s*(.*?)\"', snap, re.IGNORECASE) # Artist artists = re.findall(r'StaticText\s+\"(?:Art|Artist|Illustrated by):?\s*(.*?)\"', snap, re.IGNORECASE) # Year years = re.findall(r'StaticText\s+\"(?:Released|Published|Year):?\s*(\d{4})\"', snap, re.IGNORECASE) # Players players = re.findall(r'StaticText\s+\"(\d+[-–]\d+)\s*(?:players|Players)\"', snap) # Duration durations = re.findall(r'StaticText\s+\"(\d+)\s*(?:min|minutes|Minutes|Min\.?)\"', snap) # BGG ID bgg_ids = re.findall(r'boardgamegeek\.com/boardgame/(\d+)', snap) # Rating ratings = re.findall(r'StaticText\s+\"([\d.]+)\s*/\s*10\"', snap) # Complexity / Weight weights = re.findall(r'StaticText\s+\"([\d.]+)\s*/\s*5\"', snap) # Generic: extract all StaticText near known labels all_static = re.findall(r'StaticText\s+\"([^\"]+)\"', snap) # Look for "Number of players" pattern player_counts = re.findall(r'(\d+)\s+players', snap, re.IGNORECASE) # Premium status premium = "Premium" in snap # Number of games played games_played = re.findall(r'(\d+(?:,\d+)*)\s+games?\s+played', snap, re.IGNORECASE) print(f" Designers found: {designers[:5]}") print(f" Publishers found: {publishers[:5]}") print(f" Artists found: {artists[:5]}") print(f" Years found: {years[:5]}") print(f" Players found: {players[:5]}") print(f" Player counts: {player_counts[:5]}") print(f" Durations found: {durations[:5]}") print(f" BGG IDs: {bgg_ids[:5]}") print(f" Ratings: {ratings[:5]}") print(f" Weights: {weights[:5]}") print(f" Premium: {premium}") print(f" Games played: {games_played}") # Print first 100 static texts for manual analysis print(f" --- First 80 StaticTexts ---") for i, s in enumerate(all_static[:80]): print(f" [{i}] {s}") run(f'{AGENT_BROWSER} close --all 2>/dev/null', timeout=3) time.sleep(1) print("\nDone.")