BSN-Chatsystem/fetch_news.py

#!/usr/bin/env python3
"""Scrape Gladbach & Bundesliga news from multiple sources."""

import requests
from bs4 import BeautifulSoup
import json
import re
import sys
from urllib.parse import urljoin

HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
}

def fetch_url(url, timeout=20):
    try:
        resp = requests.get(url, headers=HEADERS, timeout=timeout)
        resp.encoding = resp.apparent_encoding or 'utf-8'
        return BeautifulSoup(resp.text, 'html.parser')
    except Exception as e:
        print(f"  ERROR fetching {url}: {e}", file=sys.stderr)
        return None

def scrape_torfabrik():
    print("=== TORFABRIK.DE ===", file=sys.stderr)
    articles = []
    soup = fetch_url("https://www.torfabrik.de/")
    if not soup:
        return articles
    
    # TYPO3 news items
    for item in soup.select(".news .article, .news-list-item, .article, .top-news, .frame-type-news_pi1 .article"):
        title_el = item.select_one("h1, h2, h3, .news-header")
        link_el = item.select_one("a")
        date_el = item.select_one("time, .date, .news-list-date")
        
        if not title_el:
            continue
        title = title_el.get_text(strip=True)
        link = None
        if link_el and link_el.get("href"):
            link = urljoin("https://www.torfabrik.de/", link_el["href"])
        elif title_el.name == "a":
            link = urljoin("https://www.torfabrik.de/", title_el.get("href", ""))
        
        date_str = date_el.get_text(strip=True) if date_el else ""
        teaser_el = item.select_one(".teaser-text, .bodytext, p")
        teaser = teaser_el.get_text(strip=True)[:300] if teaser_el else ""
        
        if title and len(title) > 10:
            articles.append({"title": title, "url": link, "date": date_str, "teaser": teaser, "source": "torfabrik.de"})
    
    # All headings with links
    for heading in soup.select("h1, h2, h3"):
        a_tag = heading.select_one("a")
        if a_tag and a_tag.get("href") and not a_tag.get("href").startswith("#"):
            text = heading.get_text(strip=True)
            if len(text) > 15:
                link = urljoin("https://www.torfabrik.de/", a_tag["href"])
                if not any(a.get("title") == text for a in articles):
                    articles.append({"title": text, "url": link, "date": "", "teaser": "", "source": "torfabrik.de"})
    
    print(f"  Found {len(articles)} articles", file=sys.stderr)
    return articles

def scrape_fohlen_hautnah():
    print("=== FOHLEN-HAUTNAH.DE ===", file=sys.stderr)
    articles = []
    
    # Try WP REST API first
    try:
        api_url = "https://fohlen-hautnah.de/wp-json/wp/v2/posts?per_page=10"
        resp = requests.get(api_url, headers=HEADERS, timeout=15)
        if resp.status_code == 200:
            posts = resp.json()
            for post in posts:
                title = post.get("title", {}).get("rendered", "")
                title = BeautifulSoup(title, "html.parser").get_text(strip=True)
                link = post.get("link", "")
                date = post.get("date", "")
                excerpt = post.get("excerpt", {}).get("rendered", "")
                excerpt = BeautifulSoup(excerpt, "html.parser").get_text(strip=True)[:300]
                content = post.get("content", {}).get("rendered", "")
                content_text = BeautifulSoup(content, "html.parser").get_text(strip=True)[:5000]
                if title:
                    articles.append({
                        "title": title, "url": link, "date": date,
                        "teaser": excerpt, "fulltext": content_text,
                        "source": "fohlen-hautnah.de"
                    })
            if articles:
                print(f"  WP API: Found {len(articles)} articles", file=sys.stderr)
                return articles
    except Exception as e:
        print(f"  WP API error: {e}", file=sys.stderr)
    
    # Fallback: scrape HTML
    soup = fetch_url("https://fohlen-hautnah.de/")
    if not soup:
        return articles
    
    for article in soup.select("article, .post, .entry"):
        title_el = article.select_one("h2 a, h3 a, .entry-title a")
        if not title_el:
            title_el = article.select_one("h2, h3, .entry-title")
        
        if not title_el:
            continue
        
        title = title_el.get_text(strip=True)
        link = title_el.get("href")
        if not link and title_el.name == "a":
            link = title_el.get("href")
        if link and not link.startswith("http"):
            link = urljoin("https://fohlen-hautnah.de/", link)
        
        date_el = article.select_one("time, .entry-date, .date")
        date_str = date_el.get("datetime") or date_el.get_text(strip=True) if date_el else ""
        teaser_el = article.select_one(".entry-summary, .excerpt, p")
        teaser = teaser_el.get_text(strip=True)[:300] if teaser_el else ""
        
        if title and len(title) > 10:
            articles.append({"title": title, "url": link, "date": date_str, "teaser": teaser, "source": "fohlen-hautnah.de"})
    
    print(f"  HTML scrape: Found {len(articles)} articles", file=sys.stderr)
    return articles

def scrape_gladbachlive():
    print("=== GLADBACHLIVE.DE ===", file=sys.stderr)
    articles = []
    
    soup = fetch_url("https://www.gladbachlive.de/")
    if not soup:
        return articles
    
    # JSON-LD structured data
    for script in soup.select("script[type='application/ld+json']"):
        try:
            data = json.loads(script.string)
            items = data if isinstance(data, list) else [data]
            for item in items:
                if isinstance(item, dict) and item.get("@type") in ("NewsArticle", "Article"):
                    articles.append({
                        "title": item.get("headline", item.get("name", "")),
                        "url": item.get("url", item.get("mainEntityOfPage", "")),
                        "date": item.get("datePublished", ""),
                        "teaser": item.get("description", "")[:300],
                        "source": "gladbachlive.de"
                    })
        except:
            pass
    
    # DM teaser cards
    for item in soup.select("[class*='teaser'], [class*='card'], [class*='article']"):
        title_el = item.select_one("h2, h3, [class*='headline'], [class*='title']")
        link_el = item.select_one("a")
        
        if not title_el:
            continue
        title = title_el.get_text(strip=True)
        link = None
        if link_el and link_el.get("href"):
            link = urljoin("https://www.gladbachlive.de/", link_el["href"])
        
        if title and len(title) > 15:
            articles.append({"title": title, "url": link, "date": "", "teaser": "", "source": "gladbachlive.de"})
    
    print(f"  Found {len(articles)} articles", file=sys.stderr)
    return articles

def scrape_kicker():
    print("=== KICKER.DE ===", file=sys.stderr)
    articles = []
    soup = fetch_url("https://www.kicker.de/bundesliga/news")
    if not soup:
        return articles
    
    for item in soup.select(".kick__item, .news-list__item, article, [class*='teaser'], [class*='news-item']"):
        title_el = item.select_one("h2, h3, .headline, .title, [class*='title'], [class*='headline']")
        link_el = item.select_one("a")
        
        if not title_el:
            continue
        title = title_el.get_text(strip=True)
        link = None
        if link_el and link_el.get("href"):
            link = urljoin("https://www.kicker.de/", link_el["href"])
        
        if title and len(title) > 15:
            articles.append({"title": title, "url": link, "date": "", "teaser": "", "source": "kicker.de"})
    
    print(f"  Found {len(articles)} articles", file=sys.stderr)
    return articles

def scrape_sportschau():
    print("=== SPORTSCHAU.DE ===", file=sys.stderr)
    articles = []
    soup = fetch_url("https://www.sportschau.de/fussball/bundesliga/")
    if not soup:
        return articles
    
    for item in soup.select("article, .teaser, [class*='teaser'], [class*='news']"):
        title_el = item.select_one("h2, h3, [class*='headline'], [class*='title']")
        link_el = item.select_one("a")
        
        if not title_el:
            continue
        title = title_el.get_text(strip=True)
        link = None
        if link_el and link_el.get("href"):
            link = urljoin("https://www.sportschau.de/", link_el["href"])
        
        if title and len(title) > 15:
            articles.append({"title": title, "url": link, "date": "", "teaser": "", "source": "sportschau.de"})
    
    print(f"  Found {len(articles)} articles", file=sys.stderr)
    return articles

def scrape_article_fulltext(url):
    if not url:
        return ""
    soup = fetch_url(url)
    if not soup:
        return ""
    
    for selector in ["article", ".article-content", ".post-content", ".entry-content",
                      ".news-content", ".article-body", ".story-body", "main", ".content", "#content"]:
        content_el = soup.select_one(selector)
        if content_el:
            for unwanted in content_el.select("script, style, nav, .ad, .advertisement, .comments, .related"):
                unwanted.decompose()
            text = content_el.get_text(separator="\n", strip=True)
            if len(text) > 200:
                return text[:5000]
    
    paragraphs = soup.select("article p, .content p, main p")
    if paragraphs:
        return "\n".join(p.get_text(strip=True) for p in paragraphs[:20])[:5000]
    return ""

def main():
    all_articles = []
    
    # Priority sources
    all_articles.extend(scrape_torfabrik())
    all_articles.extend(scrape_fohlen_hautnah())
    all_articles.extend(scrape_gladbachlive())
    
    # Deduplicate
    seen = set()
    unique = []
    for a in all_articles:
        key = a["title"].lower().strip()[:80]
        if key and key not in seen:
            seen.add(key)
            unique.append(a)
    
    print(f"\n=== TOTAL UNIQUE: {len(unique)} articles ===", file=sys.stderr)
    
    # Scrape full text for top articles
    for a in unique[:15]:
        if a.get("fulltext"):
            continue  # already have it from API
        url = a.get("url")
        if url:
            print(f"Fetching fulltext: {url}", file=sys.stderr)
            a["fulltext"] = scrape_article_fulltext(url)
    
    # Save to JSON
    with open("/home/hermes/workspace/scraped_articles.json", "w") as f:
        json.dump(unique, f, ensure_ascii=False, indent=2)
    
    # Print summary
    for i, a in enumerate(unique[:30]):
        print(f"[{i+1}] [{a['source']}] {a['title']}")
        if a.get('date'):
            print(f"    Date: {a['date']}")
        if a.get('url'):
            print(f"    URL: {a['url']}")
        if a.get('teaser'):
            print(f"    Teaser: {a['teaser'][:200]}")
        if a.get('fulltext'):
            print(f"    Fulltext: {len(a['fulltext'])} chars")
        print()

if __name__ == "__main__":
    main()