fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)
This commit is contained in:
Hermes Agent
+289
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrape Gladbach & Bundesliga news from multiple sources."""
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urljoin
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
def fetch_url(url, timeout=20):
|
||||
try:
|
||||
resp = requests.get(url, headers=HEADERS, timeout=timeout)
|
||||
resp.encoding = resp.apparent_encoding or 'utf-8'
|
||||
return BeautifulSoup(resp.text, 'html.parser')
|
||||
except Exception as e:
|
||||
print(f" ERROR fetching {url}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
def scrape_torfabrik():
|
||||
print("=== TORFABRIK.DE ===", file=sys.stderr)
|
||||
articles = []
|
||||
soup = fetch_url("https://www.torfabrik.de/")
|
||||
if not soup:
|
||||
return articles
|
||||
|
||||
# TYPO3 news items
|
||||
for item in soup.select(".news .article, .news-list-item, .article, .top-news, .frame-type-news_pi1 .article"):
|
||||
title_el = item.select_one("h1, h2, h3, .news-header")
|
||||
link_el = item.select_one("a")
|
||||
date_el = item.select_one("time, .date, .news-list-date")
|
||||
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
link = None
|
||||
if link_el and link_el.get("href"):
|
||||
link = urljoin("https://www.torfabrik.de/", link_el["href"])
|
||||
elif title_el.name == "a":
|
||||
link = urljoin("https://www.torfabrik.de/", title_el.get("href", ""))
|
||||
|
||||
date_str = date_el.get_text(strip=True) if date_el else ""
|
||||
teaser_el = item.select_one(".teaser-text, .bodytext, p")
|
||||
teaser = teaser_el.get_text(strip=True)[:300] if teaser_el else ""
|
||||
|
||||
if title and len(title) > 10:
|
||||
articles.append({"title": title, "url": link, "date": date_str, "teaser": teaser, "source": "torfabrik.de"})
|
||||
|
||||
# All headings with links
|
||||
for heading in soup.select("h1, h2, h3"):
|
||||
a_tag = heading.select_one("a")
|
||||
if a_tag and a_tag.get("href") and not a_tag.get("href").startswith("#"):
|
||||
text = heading.get_text(strip=True)
|
||||
if len(text) > 15:
|
||||
link = urljoin("https://www.torfabrik.de/", a_tag["href"])
|
||||
if not any(a.get("title") == text for a in articles):
|
||||
articles.append({"title": text, "url": link, "date": "", "teaser": "", "source": "torfabrik.de"})
|
||||
|
||||
print(f" Found {len(articles)} articles", file=sys.stderr)
|
||||
return articles
|
||||
|
||||
def scrape_fohlen_hautnah():
|
||||
print("=== FOHLEN-HAUTNAH.DE ===", file=sys.stderr)
|
||||
articles = []
|
||||
|
||||
# Try WP REST API first
|
||||
try:
|
||||
api_url = "https://fohlen-hautnah.de/wp-json/wp/v2/posts?per_page=10"
|
||||
resp = requests.get(api_url, headers=HEADERS, timeout=15)
|
||||
if resp.status_code == 200:
|
||||
posts = resp.json()
|
||||
for post in posts:
|
||||
title = post.get("title", {}).get("rendered", "")
|
||||
title = BeautifulSoup(title, "html.parser").get_text(strip=True)
|
||||
link = post.get("link", "")
|
||||
date = post.get("date", "")
|
||||
excerpt = post.get("excerpt", {}).get("rendered", "")
|
||||
excerpt = BeautifulSoup(excerpt, "html.parser").get_text(strip=True)[:300]
|
||||
content = post.get("content", {}).get("rendered", "")
|
||||
content_text = BeautifulSoup(content, "html.parser").get_text(strip=True)[:5000]
|
||||
if title:
|
||||
articles.append({
|
||||
"title": title, "url": link, "date": date,
|
||||
"teaser": excerpt, "fulltext": content_text,
|
||||
"source": "fohlen-hautnah.de"
|
||||
})
|
||||
if articles:
|
||||
print(f" WP API: Found {len(articles)} articles", file=sys.stderr)
|
||||
return articles
|
||||
except Exception as e:
|
||||
print(f" WP API error: {e}", file=sys.stderr)
|
||||
|
||||
# Fallback: scrape HTML
|
||||
soup = fetch_url("https://fohlen-hautnah.de/")
|
||||
if not soup:
|
||||
return articles
|
||||
|
||||
for article in soup.select("article, .post, .entry"):
|
||||
title_el = article.select_one("h2 a, h3 a, .entry-title a")
|
||||
if not title_el:
|
||||
title_el = article.select_one("h2, h3, .entry-title")
|
||||
|
||||
if not title_el:
|
||||
continue
|
||||
|
||||
title = title_el.get_text(strip=True)
|
||||
link = title_el.get("href")
|
||||
if not link and title_el.name == "a":
|
||||
link = title_el.get("href")
|
||||
if link and not link.startswith("http"):
|
||||
link = urljoin("https://fohlen-hautnah.de/", link)
|
||||
|
||||
date_el = article.select_one("time, .entry-date, .date")
|
||||
date_str = date_el.get("datetime") or date_el.get_text(strip=True) if date_el else ""
|
||||
teaser_el = article.select_one(".entry-summary, .excerpt, p")
|
||||
teaser = teaser_el.get_text(strip=True)[:300] if teaser_el else ""
|
||||
|
||||
if title and len(title) > 10:
|
||||
articles.append({"title": title, "url": link, "date": date_str, "teaser": teaser, "source": "fohlen-hautnah.de"})
|
||||
|
||||
print(f" HTML scrape: Found {len(articles)} articles", file=sys.stderr)
|
||||
return articles
|
||||
|
||||
def scrape_gladbachlive():
|
||||
print("=== GLADBACHLIVE.DE ===", file=sys.stderr)
|
||||
articles = []
|
||||
|
||||
soup = fetch_url("https://www.gladbachlive.de/")
|
||||
if not soup:
|
||||
return articles
|
||||
|
||||
# JSON-LD structured data
|
||||
for script in soup.select("script[type='application/ld+json']"):
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for item in items:
|
||||
if isinstance(item, dict) and item.get("@type") in ("NewsArticle", "Article"):
|
||||
articles.append({
|
||||
"title": item.get("headline", item.get("name", "")),
|
||||
"url": item.get("url", item.get("mainEntityOfPage", "")),
|
||||
"date": item.get("datePublished", ""),
|
||||
"teaser": item.get("description", "")[:300],
|
||||
"source": "gladbachlive.de"
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
# DM teaser cards
|
||||
for item in soup.select("[class*='teaser'], [class*='card'], [class*='article']"):
|
||||
title_el = item.select_one("h2, h3, [class*='headline'], [class*='title']")
|
||||
link_el = item.select_one("a")
|
||||
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
link = None
|
||||
if link_el and link_el.get("href"):
|
||||
link = urljoin("https://www.gladbachlive.de/", link_el["href"])
|
||||
|
||||
if title and len(title) > 15:
|
||||
articles.append({"title": title, "url": link, "date": "", "teaser": "", "source": "gladbachlive.de"})
|
||||
|
||||
print(f" Found {len(articles)} articles", file=sys.stderr)
|
||||
return articles
|
||||
|
||||
def scrape_kicker():
|
||||
print("=== KICKER.DE ===", file=sys.stderr)
|
||||
articles = []
|
||||
soup = fetch_url("https://www.kicker.de/bundesliga/news")
|
||||
if not soup:
|
||||
return articles
|
||||
|
||||
for item in soup.select(".kick__item, .news-list__item, article, [class*='teaser'], [class*='news-item']"):
|
||||
title_el = item.select_one("h2, h3, .headline, .title, [class*='title'], [class*='headline']")
|
||||
link_el = item.select_one("a")
|
||||
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
link = None
|
||||
if link_el and link_el.get("href"):
|
||||
link = urljoin("https://www.kicker.de/", link_el["href"])
|
||||
|
||||
if title and len(title) > 15:
|
||||
articles.append({"title": title, "url": link, "date": "", "teaser": "", "source": "kicker.de"})
|
||||
|
||||
print(f" Found {len(articles)} articles", file=sys.stderr)
|
||||
return articles
|
||||
|
||||
def scrape_sportschau():
|
||||
print("=== SPORTSCHAU.DE ===", file=sys.stderr)
|
||||
articles = []
|
||||
soup = fetch_url("https://www.sportschau.de/fussball/bundesliga/")
|
||||
if not soup:
|
||||
return articles
|
||||
|
||||
for item in soup.select("article, .teaser, [class*='teaser'], [class*='news']"):
|
||||
title_el = item.select_one("h2, h3, [class*='headline'], [class*='title']")
|
||||
link_el = item.select_one("a")
|
||||
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
link = None
|
||||
if link_el and link_el.get("href"):
|
||||
link = urljoin("https://www.sportschau.de/", link_el["href"])
|
||||
|
||||
if title and len(title) > 15:
|
||||
articles.append({"title": title, "url": link, "date": "", "teaser": "", "source": "sportschau.de"})
|
||||
|
||||
print(f" Found {len(articles)} articles", file=sys.stderr)
|
||||
return articles
|
||||
|
||||
def scrape_article_fulltext(url):
|
||||
if not url:
|
||||
return ""
|
||||
soup = fetch_url(url)
|
||||
if not soup:
|
||||
return ""
|
||||
|
||||
for selector in ["article", ".article-content", ".post-content", ".entry-content",
|
||||
".news-content", ".article-body", ".story-body", "main", ".content", "#content"]:
|
||||
content_el = soup.select_one(selector)
|
||||
if content_el:
|
||||
for unwanted in content_el.select("script, style, nav, .ad, .advertisement, .comments, .related"):
|
||||
unwanted.decompose()
|
||||
text = content_el.get_text(separator="\n", strip=True)
|
||||
if len(text) > 200:
|
||||
return text[:5000]
|
||||
|
||||
paragraphs = soup.select("article p, .content p, main p")
|
||||
if paragraphs:
|
||||
return "\n".join(p.get_text(strip=True) for p in paragraphs[:20])[:5000]
|
||||
return ""
|
||||
|
||||
def main():
|
||||
all_articles = []
|
||||
|
||||
# Priority sources
|
||||
all_articles.extend(scrape_torfabrik())
|
||||
all_articles.extend(scrape_fohlen_hautnah())
|
||||
all_articles.extend(scrape_gladbachlive())
|
||||
|
||||
# Deduplicate
|
||||
seen = set()
|
||||
unique = []
|
||||
for a in all_articles:
|
||||
key = a["title"].lower().strip()[:80]
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(a)
|
||||
|
||||
print(f"\n=== TOTAL UNIQUE: {len(unique)} articles ===", file=sys.stderr)
|
||||
|
||||
# Scrape full text for top articles
|
||||
for a in unique[:15]:
|
||||
if a.get("fulltext"):
|
||||
continue # already have it from API
|
||||
url = a.get("url")
|
||||
if url:
|
||||
print(f"Fetching fulltext: {url}", file=sys.stderr)
|
||||
a["fulltext"] = scrape_article_fulltext(url)
|
||||
|
||||
# Save to JSON
|
||||
with open("/home/hermes/workspace/scraped_articles.json", "w") as f:
|
||||
json.dump(unique, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# Print summary
|
||||
for i, a in enumerate(unique[:30]):
|
||||
print(f"[{i+1}] [{a['source']}] {a['title']}")
|
||||
if a.get('date'):
|
||||
print(f" Date: {a['date']}")
|
||||
if a.get('url'):
|
||||
print(f" URL: {a['url']}")
|
||||
if a.get('teaser'):
|
||||
print(f" Teaser: {a['teaser'][:200]}")
|
||||
if a.get('fulltext'):
|
||||
print(f" Fulltext: {len(a['fulltext'])} chars")
|
||||
print()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user