import urllib.request, json, sys

TAVILY_API_KEY = "tvly-dev-REPLACE_ME"  # We'll try a direct approach

# Let's first try to find if tavily is installed somewhere
import importlib
try:
    import tavily
    print("tavily available")
except ImportError:
    print("tavily not available")

# Let's try direct web scraping of known sources
urls = {
    "asmodee_study": [
        "https://www.dicebreaker.com/companies/asmodee/news/asmodee-kantar-board-games-mental-health-study",
        "https://www.tabletopgaming.co.uk/news/asmodee-and-kantar-study-finds-board-games-boost-mental-health",
    ],
    "steamforged": [
        "https://www.dicebreaker.com/companies/steamforged-games/news/steamforged-games-layoffs-2025",
        "https://www.polygon.com/tabletop-games/steamforged-games-layoffs",
    ],
    "dire_wolf": [
        "https://www.direwolfdigital.com/news/",
        "https://boardgamegeek.com/boardgame/428612/trains/credits",
    ]
}

for category, urllist in urls.items():
    print(f"\n=== {category} ===")
    for url in urllist:
        try:
            req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'})
            with urllib.request.urlopen(req, timeout=15) as resp:
                html = resp.read().decode('utf-8', errors='replace')
                print(f"URL: {url} - Status: {resp.status} - Size: {len(html)}")
                # Extract title
                import re
                title_match = re.search(r'<title>(.*?)</title>', html, re.IGNORECASE)
                if title_match:
                    print(f"  Title: {title_match.group(1)}")
        except Exception as e:
            print(f"URL: {url} - Error: {e}")