import requests
from bs4 import BeautifulSoup
import json
import sys

HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

def fetch_url(url, timeout=20):
    try:
        resp = requests.get(url, headers=HEADERS, timeout=timeout)
        resp.encoding = resp.apparent_encoding or 'utf-8'
        return resp.text
    except Exception as e:
        print(f"ERROR: {url}: {e}", file=sys.stderr)
        return None

def get_fulltext(html):
    soup = BeautifulSoup(html, 'html.parser')
    for selector in ["article", ".article-content", ".post-content", ".entry-content",
                      ".news-content", ".article-body", ".story-body", "main", ".content", "#content"]:
        content_el = soup.select_one(selector)
        if content_el:
            for unwanted in content_el.select("script, style, nav, .ad, .advertisement, .comments, .related"):
                unwanted.decompose()
            text = content_el.get_text(separator="\n", strip=True)
            if len(text) > 200:
                return text[:5000]
    paragraphs = soup.select("article p, .content p, main p")
    if paragraphs:
        return "\n".join(p.get_text(strip=True) for p in paragraphs[:20])[:5000]
    return ""

# Fetch remaining gladbachlive articles
urls = [
    ("Wird der WM-Traum wahr? Gladbach-Star vor Duell mit Fohlen-Kollegen",
     "https://www.gladbachlive.de/news/wird-der-wm-traum-wahr-gladbach-star-vor-duell-mit-fohlen-kollegen-1296117"),
    ("Große Begehrlichkeit geweckt: Borussia-Talent vor dem Absprung?",
     "https://www.gladbachlive.de/news/grosse-begehrlichkeit-geweckt-borussia-talent-vor-dem-absprung-1295909"),
    ("Gladbacher vor der WM: Elvedi-Truppe patzt",
     "https://www.gladbachlive.de/news/gladbacher-vor-der-wm-elvedi-truppe-patzt-gegen-aussenseiter-1295352"),
    ("Fohlenelf vor machbarer Aufgabe im DFB-Pokal",
     "https://www.gladbachlive.de/news/in-bundesliga-stadion-gladbach-vor-machbarer-aufgabe-im-dfb-pokal-1295280"),
    ("Hüne aus Belgien als Verstärkung für die Abwehr?",
     "https://www.gladbachlive.de/news/geruechte-in-gladbach-abwehr-huene-als-defensiv-verstaerkung-1295178"),
]

results = {}
for title, url in urls:
    print(f"Fetching: {url}", file=sys.stderr)
    html = fetch_url(url)
    if html:
        ft = get_fulltext(html)
        results[url] = {"title": title, "fulltext": ft, "length": len(ft)}
        print(f"  Got {len(ft)} chars", file=sys.stderr)

# Also get TorFabrik article
torfabrik_url = "https://www.torfabrik.de/"
html = fetch_url(torfabrik_url)
if html:
    soup = BeautifulSoup(html, 'html.parser')
    # Find the top article link
    top_news = soup.select_one(".top-news a") or soup.select_one("h1 a, .news-header a")
    if top_news and top_news.get("href"):
        article_url = "https://www.torfabrik.de" + top_news["href"] if not top_news["href"].startswith("http") else top_news["href"]
        print(f"Fetching TorFabrik article: {article_url}", file=sys.stderr)
        article_html = fetch_url(article_url)
        if article_html:
            ft = get_fulltext(article_html)
            results[article_url] = {"title": "Borussia verpflichtet Isac Lidberg", "fulltext": ft, "length": len(ft)}
            print(f"  Got {len(ft)} chars", file=sys.stderr)

# Save results
with open("/home/hermes/workspace/more_fulltext.json", "w") as f:
    json.dump(results, f, ensure_ascii=False, indent=2)

# Print summary
for url, data in results.items():
    print(f"\n{'='*60}")
    print(f"TITLE: {data['title']}")
    print(f"URL: {url}")
    print(f"LENGTH: {data['length']} chars")
    print(f"TEXT:\n{data['fulltext'][:1000]}")