#!/usr/bin/env python3
"""Scrape Gladbach-focused football news from German sources."""

import requests
from bs4 import BeautifulSoup
import json
import time
import re

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
}

GLADBACH_KW = [
    'gladbach', 'borussia', 'mönchengladbach', 'fohlen', 'fohlenelf',
    'borussia park', 'niederrhein', 'borussen',
    'elvedi', 'plea', 'honorat', 'neuhaus', 'weigl', 'koné',
    'scally', 'oultara', 'friedrich', 'itakura', 'ngoumou',
    'cvančara', 'civancara', 'ranos', 'hack',
    'reitz', 'netz', 'wöber', 'woeber', 'kleindienst',
    'seoane', 'virkus',
]

def fetch_url(url, retries=2):
    for attempt in range(retries + 1):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=20)
            resp.raise_for_status()
            resp.encoding = resp.apparent_encoding or 'utf-8'
            return resp
        except Exception as e:
            if attempt < retries:
                time.sleep(2)
            else:
                print(f"  FAILED: {url} — {e}")
                return None

def clean_text(text):
    return re.sub(r'\s+', ' ', text).strip()

def is_recent_date(date_str):
    patterns = [
        r'01\.06\.2026', r'02\.06\.2026',
        r'1\.\s*Juni\s*2026', r'2\.\s*Juni\s*2026',
        r'01\.06\.26', r'02\.06\.26',
        r'2026-06-01', r'2026-06-02',
        r'vor \d+ Stunden', r'vor \d+ Minuten',
    ]
    for pat in patterns:
        if re.search(pat, date_str, re.IGNORECASE):
            return True
    return False

def has_gladbach_keyword(text, min_hits=1):
    text_lower = text.lower()
    hits = sum(1 for kw in GLADBACH_KW if kw in text_lower)
    return hits >= min_hits

def extract_article_text(soup):
    selectors = [
        'article', '.article-content', '.article-body', '.post-content',
        '.entry-content', '.content', '.story-body', '.article__body',
        '.article-text', '.news-text', 'main', '.main-content',
        '[itemprop="articleBody"]', '.article__content', '.post__content',
        '.single-content', '#article-content',
    ]
    for sel in selectors:
        elem = soup.select_one(sel)
        if elem:
            for unwanted in elem.find_all(['script', 'style', 'nav', 'aside', 'footer', 'header', 'iframe', 'form']):
                unwanted.decompose()
            text = elem.get_text(separator=' ', strip=True)
            text = clean_text(text)
            if len(text) > 200:
                return text
    paragraphs = soup.find_all('p')
    texts = [clean_text(p.get_text()) for p in paragraphs if len(p.get_text().strip()) > 50]
    if texts:
        return ' '.join(texts)
    body = soup.find('body')
    if body:
        for unwanted in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'iframe']):
            unwanted.decompose()
        return clean_text(body.get_text(separator=' ', strip=True))
    return ""

def extract_date(soup):
    date_patterns = [
        ('meta[property="article:published_time"]', 'content'),
        ('meta[name="date"]', 'content'),
        ('meta[name="pubdate"]', 'content'),
        ('time', 'datetime'),
        ('time[datetime]', 'datetime'),
    ]
    for selector, attr in date_patterns:
        elem = soup.select_one(selector)
        if elem:
            val = elem.get(attr, '') if attr else elem.get_text(strip=True)
            if val and len(val) > 5:
                return val
    text = soup.get_text()
    m = re.search(r'(\d{1,2}\.\s*\d{1,2}\.\s*(?:20)?2[456])', text)
    return m.group(1) if m else ""

def generic_scrape(name, base_urls, url_filter=None, title_cleanup=None, gladbach_check=True):
    """Generic scraper for a news site."""
    print(f"\n{'='*60}\nSCRAPING: {name}\n{'='*60}")
    results = []
    
    for base_url in base_urls:
        print(f"  Trying: {base_url}")
        resp = fetch_url(base_url)
        if not resp:
            continue
        
        soup = BeautifulSoup(resp.text, 'html.parser')
        links = soup.find_all('a', href=True)
        article_urls = set()
        
        for link in links:
            href = link['href']
            if not href or href.startswith('#') or href.startswith('javascript:') or href.startswith('mailto:'):
                continue
            if href.endswith(('.jpg', '.png', '.gif', '.pdf', '.css', '.js', '.ico')):
                continue
            
            # Build full URL
            if href.startswith('http'):
                full_url = href
            elif href.startswith('//'):
                full_url = 'https:' + href
            elif href.startswith('/'):
                # Extract domain from base_url
                domain = re.match(r'https?://[^/]+', base_url)
                if domain:
                    full_url = domain.group(0) + href
                else:
                    continue
            else:
                continue
            
            if full_url in article_urls:
                continue
            
            # Apply filter if provided
            if url_filter:
                if url_filter(full_url):
                    article_urls.add(full_url)
            else:
                article_urls.add(full_url)
        
        print(f"  Found {len(article_urls)} potential article URLs (sampling first 30)")
        
        for art_url in list(article_urls)[:30]:
            print(f"    Fetching: {art_url}")
            art_resp = fetch_url(art_url)
            if not art_resp:
                continue
            
            art_soup = BeautifulSoup(art_resp.text, 'html.parser')
            
            # Extract title
            title = None
            title_elem = art_soup.find('title')
            if title_elem:
                title = clean_text(title_elem.get_text())
                if title_cleanup:
                    title = title_cleanup(title)
            if not title or len(title) < 15:
                h1 = art_soup.find('h1')
                if h1:
                    title = clean_text(h1.get_text())
            if not title or len(title) < 15:
                continue
            
            date_str = extract_date(art_soup)
            full_text = extract_article_text(art_soup)
            
            if not full_text or len(full_text) < 100:
                print(f"      Skipping (no usable text)")
                continue
            
            if date_str and not is_recent_date(date_str):
                print(f"      Skipping (date: {date_str})")
                continue
            
            if gladbach_check:
                combined = (title + ' ' + full_text).lower()
                if not has_gladbach_keyword(combined, min_hits=1):
                    print(f"      Not Gladbach-related, skipping")
                    continue
            
            print(f"      ✓ FOUND: {title[:80]}")
            results.append({
                'source': name,
                'title': title,
                'url': art_url,
                'date': date_str or 'Kein Datum gefunden',
                'full_text': full_text[:3000],
            })
            
            if len(results) >= 2:
                break
        
        if results:
            break
    
    return results

def main():
    all_results = []
    
    # 1. torfabrik.de
    all_results.extend(generic_scrape(
        'torfabrik.de',
        ['https://www.torfabrik.de/', 'https://www.torfabrik.de/borussia-moenchengladbach/'],
        url_filter=lambda u: 'torfabrik.de' in u,
        title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Tt]orfabrik.*$', '', t).strip(),
    ))
    
    # 2. fohlen-hautnah.de
    all_results.extend(generic_scrape(
        'fohlen-hautnah.de',
        ['https://www.fohlen-hautnah.de/'],
        url_filter=lambda u: 'fohlen-hautnah.de' in u,
        title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ff]ohlen[-\s]*[Hh]autnah.*$', '', t).strip(),
        gladbach_check=False,  # All articles are Gladbach-related
    ))
    
    # 3. gladbachlive.de
    all_results.extend(generic_scrape(
        'gladbachlive.de',
        ['https://www.gladbachlive.de/', 'https://www.gladbachlive.de/news/'],
        url_filter=lambda u: 'gladbachlive.de' in u,
        title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Gg]ladbach[Ll]ive.*$', '', t).strip(),
    ))
    
    # 4. kicker.de
    all_results.extend(generic_scrape(
        'kicker.de',
        ['https://www.kicker.de/borussia-moenchengladbach/news', 'https://www.kicker.de/bundesliga/aktuell'],
        url_filter=lambda u: 'kicker.de' in u and any(kw in u.lower() for kw in ['gladbach', 'borussia-mg', 'bundesliga', 'news', 'transfer']),
        title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Kk]icker.*$', '', t).strip(),
    ))
    
    # 5. sportschau.de (general Bundesliga, no strict Gladbach filter)
    all_results.extend(generic_scrape(
        'sportschau.de',
        ['https://www.sportschau.de/fussball/bundesliga/'],
        url_filter=lambda u: 'sportschau.de' in u and '/fussball/' in u,
        title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ss]portschau.*$', '', t).strip(),
        gladbach_check=False,
    ))
    
    # 6. fussballtransfers.com
    all_results.extend(generic_scrape(
        'fussballtransfers.com',
        ['https://www.fussballtransfers.com/bundesliga', 'https://www.fussballtransfers.com/'],
        url_filter=lambda u: 'fussballtransfers.com' in u,
        title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ff]ussball\s*[Tt]ransfers.*$', '', t).strip(),
    ))
    
    # Deduplicate by URL
    seen_urls = set()
    deduped = []
    for r in all_results:
        if r['url'] not in seen_urls:
            seen_urls.add(r['url'])
            deduped.append(r)
    
    # Prioritize: Gladbach articles first (with 2+ keyword hits), then the rest
    gladbach_heavy = [r for r in deduped if has_gladbach_keyword(r['title'] + ' ' + r['full_text'], min_hits=2)]
    gladbach_light = [r for r in deduped if r not in gladbach_heavy and has_gladbach_keyword(r['title'] + ' ' + r['full_text'], min_hits=1)]
    other = [r for r in deduped if r not in gladbach_heavy and r not in gladbach_light]
    
    # Take 5: prefer heavy Gladbach, then light Gladbach, then other
    final = gladbach_heavy + gladbach_light + other
    final = final[:5]
    
    print("\n\n" + "=" * 60)
    print(f"FINAL RESULTS: {len(final)} articles")
    print("=" * 60)
    
    for i, r in enumerate(final, 1):
        print(f"\n--- Artikel {i} ---")
        print(f"Quelle: {r['source']}")
        print(f"Titel: {r['title']}")
        print(f"Datum: {r['date']}")
        print(f"URL: {r['url']}")
        print(f"Textlänge: {len(r['full_text'])} Zeichen")
    
    with open('/home/hermes/workspace/gladbach_news.json', 'w', encoding='utf-8') as f:
        json.dump(final, f, ensure_ascii=False, indent=2)
    
    print(f"\nResults saved to /home/hermes/workspace/gladbach_news.json")
    return final

if __name__ == '__main__':
    main()