fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)
This commit is contained in:
Hermes Agent
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrape Gladbach-focused football news from German sources."""
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import time
|
||||
import re
|
||||
|
||||
HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
}
|
||||
|
||||
GLADBACH_KW = [
|
||||
'gladbach', 'borussia', 'mönchengladbach', 'fohlen', 'fohlenelf',
|
||||
'borussia park', 'niederrhein', 'borussen',
|
||||
'elvedi', 'plea', 'honorat', 'neuhaus', 'weigl', 'koné',
|
||||
'scally', 'oultara', 'friedrich', 'itakura', 'ngoumou',
|
||||
'cvančara', 'civancara', 'ranos', 'hack',
|
||||
'reitz', 'netz', 'wöber', 'woeber', 'kleindienst',
|
||||
'seoane', 'virkus',
|
||||
]
|
||||
|
||||
def fetch_url(url, retries=2):
|
||||
for attempt in range(retries + 1):
|
||||
try:
|
||||
resp = requests.get(url, headers=HEADERS, timeout=20)
|
||||
resp.raise_for_status()
|
||||
resp.encoding = resp.apparent_encoding or 'utf-8'
|
||||
return resp
|
||||
except Exception as e:
|
||||
if attempt < retries:
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(f" FAILED: {url} — {e}")
|
||||
return None
|
||||
|
||||
def clean_text(text):
|
||||
return re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
def is_recent_date(date_str):
|
||||
patterns = [
|
||||
r'01\.06\.2026', r'02\.06\.2026',
|
||||
r'1\.\s*Juni\s*2026', r'2\.\s*Juni\s*2026',
|
||||
r'01\.06\.26', r'02\.06\.26',
|
||||
r'2026-06-01', r'2026-06-02',
|
||||
r'vor \d+ Stunden', r'vor \d+ Minuten',
|
||||
]
|
||||
for pat in patterns:
|
||||
if re.search(pat, date_str, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
def has_gladbach_keyword(text, min_hits=1):
|
||||
text_lower = text.lower()
|
||||
hits = sum(1 for kw in GLADBACH_KW if kw in text_lower)
|
||||
return hits >= min_hits
|
||||
|
||||
def extract_article_text(soup):
|
||||
selectors = [
|
||||
'article', '.article-content', '.article-body', '.post-content',
|
||||
'.entry-content', '.content', '.story-body', '.article__body',
|
||||
'.article-text', '.news-text', 'main', '.main-content',
|
||||
'[itemprop="articleBody"]', '.article__content', '.post__content',
|
||||
'.single-content', '#article-content',
|
||||
]
|
||||
for sel in selectors:
|
||||
elem = soup.select_one(sel)
|
||||
if elem:
|
||||
for unwanted in elem.find_all(['script', 'style', 'nav', 'aside', 'footer', 'header', 'iframe', 'form']):
|
||||
unwanted.decompose()
|
||||
text = elem.get_text(separator=' ', strip=True)
|
||||
text = clean_text(text)
|
||||
if len(text) > 200:
|
||||
return text
|
||||
paragraphs = soup.find_all('p')
|
||||
texts = [clean_text(p.get_text()) for p in paragraphs if len(p.get_text().strip()) > 50]
|
||||
if texts:
|
||||
return ' '.join(texts)
|
||||
body = soup.find('body')
|
||||
if body:
|
||||
for unwanted in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'iframe']):
|
||||
unwanted.decompose()
|
||||
return clean_text(body.get_text(separator=' ', strip=True))
|
||||
return ""
|
||||
|
||||
def extract_date(soup):
|
||||
date_patterns = [
|
||||
('meta[property="article:published_time"]', 'content'),
|
||||
('meta[name="date"]', 'content'),
|
||||
('meta[name="pubdate"]', 'content'),
|
||||
('time', 'datetime'),
|
||||
('time[datetime]', 'datetime'),
|
||||
]
|
||||
for selector, attr in date_patterns:
|
||||
elem = soup.select_one(selector)
|
||||
if elem:
|
||||
val = elem.get(attr, '') if attr else elem.get_text(strip=True)
|
||||
if val and len(val) > 5:
|
||||
return val
|
||||
text = soup.get_text()
|
||||
m = re.search(r'(\d{1,2}\.\s*\d{1,2}\.\s*(?:20)?2[456])', text)
|
||||
return m.group(1) if m else ""
|
||||
|
||||
def generic_scrape(name, base_urls, url_filter=None, title_cleanup=None, gladbach_check=True):
|
||||
"""Generic scraper for a news site."""
|
||||
print(f"\n{'='*60}\nSCRAPING: {name}\n{'='*60}")
|
||||
results = []
|
||||
|
||||
for base_url in base_urls:
|
||||
print(f" Trying: {base_url}")
|
||||
resp = fetch_url(base_url)
|
||||
if not resp:
|
||||
continue
|
||||
|
||||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||||
links = soup.find_all('a', href=True)
|
||||
article_urls = set()
|
||||
|
||||
for link in links:
|
||||
href = link['href']
|
||||
if not href or href.startswith('#') or href.startswith('javascript:') or href.startswith('mailto:'):
|
||||
continue
|
||||
if href.endswith(('.jpg', '.png', '.gif', '.pdf', '.css', '.js', '.ico')):
|
||||
continue
|
||||
|
||||
# Build full URL
|
||||
if href.startswith('http'):
|
||||
full_url = href
|
||||
elif href.startswith('//'):
|
||||
full_url = 'https:' + href
|
||||
elif href.startswith('/'):
|
||||
# Extract domain from base_url
|
||||
domain = re.match(r'https?://[^/]+', base_url)
|
||||
if domain:
|
||||
full_url = domain.group(0) + href
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
if full_url in article_urls:
|
||||
continue
|
||||
|
||||
# Apply filter if provided
|
||||
if url_filter:
|
||||
if url_filter(full_url):
|
||||
article_urls.add(full_url)
|
||||
else:
|
||||
article_urls.add(full_url)
|
||||
|
||||
print(f" Found {len(article_urls)} potential article URLs (sampling first 30)")
|
||||
|
||||
for art_url in list(article_urls)[:30]:
|
||||
print(f" Fetching: {art_url}")
|
||||
art_resp = fetch_url(art_url)
|
||||
if not art_resp:
|
||||
continue
|
||||
|
||||
art_soup = BeautifulSoup(art_resp.text, 'html.parser')
|
||||
|
||||
# Extract title
|
||||
title = None
|
||||
title_elem = art_soup.find('title')
|
||||
if title_elem:
|
||||
title = clean_text(title_elem.get_text())
|
||||
if title_cleanup:
|
||||
title = title_cleanup(title)
|
||||
if not title or len(title) < 15:
|
||||
h1 = art_soup.find('h1')
|
||||
if h1:
|
||||
title = clean_text(h1.get_text())
|
||||
if not title or len(title) < 15:
|
||||
continue
|
||||
|
||||
date_str = extract_date(art_soup)
|
||||
full_text = extract_article_text(art_soup)
|
||||
|
||||
if not full_text or len(full_text) < 100:
|
||||
print(f" Skipping (no usable text)")
|
||||
continue
|
||||
|
||||
if date_str and not is_recent_date(date_str):
|
||||
print(f" Skipping (date: {date_str})")
|
||||
continue
|
||||
|
||||
if gladbach_check:
|
||||
combined = (title + ' ' + full_text).lower()
|
||||
if not has_gladbach_keyword(combined, min_hits=1):
|
||||
print(f" Not Gladbach-related, skipping")
|
||||
continue
|
||||
|
||||
print(f" ✓ FOUND: {title[:80]}")
|
||||
results.append({
|
||||
'source': name,
|
||||
'title': title,
|
||||
'url': art_url,
|
||||
'date': date_str or 'Kein Datum gefunden',
|
||||
'full_text': full_text[:3000],
|
||||
})
|
||||
|
||||
if len(results) >= 2:
|
||||
break
|
||||
|
||||
if results:
|
||||
break
|
||||
|
||||
return results
|
||||
|
||||
def main():
|
||||
all_results = []
|
||||
|
||||
# 1. torfabrik.de
|
||||
all_results.extend(generic_scrape(
|
||||
'torfabrik.de',
|
||||
['https://www.torfabrik.de/', 'https://www.torfabrik.de/borussia-moenchengladbach/'],
|
||||
url_filter=lambda u: 'torfabrik.de' in u,
|
||||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Tt]orfabrik.*$', '', t).strip(),
|
||||
))
|
||||
|
||||
# 2. fohlen-hautnah.de
|
||||
all_results.extend(generic_scrape(
|
||||
'fohlen-hautnah.de',
|
||||
['https://www.fohlen-hautnah.de/'],
|
||||
url_filter=lambda u: 'fohlen-hautnah.de' in u,
|
||||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ff]ohlen[-\s]*[Hh]autnah.*$', '', t).strip(),
|
||||
gladbach_check=False, # All articles are Gladbach-related
|
||||
))
|
||||
|
||||
# 3. gladbachlive.de
|
||||
all_results.extend(generic_scrape(
|
||||
'gladbachlive.de',
|
||||
['https://www.gladbachlive.de/', 'https://www.gladbachlive.de/news/'],
|
||||
url_filter=lambda u: 'gladbachlive.de' in u,
|
||||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Gg]ladbach[Ll]ive.*$', '', t).strip(),
|
||||
))
|
||||
|
||||
# 4. kicker.de
|
||||
all_results.extend(generic_scrape(
|
||||
'kicker.de',
|
||||
['https://www.kicker.de/borussia-moenchengladbach/news', 'https://www.kicker.de/bundesliga/aktuell'],
|
||||
url_filter=lambda u: 'kicker.de' in u and any(kw in u.lower() for kw in ['gladbach', 'borussia-mg', 'bundesliga', 'news', 'transfer']),
|
||||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Kk]icker.*$', '', t).strip(),
|
||||
))
|
||||
|
||||
# 5. sportschau.de (general Bundesliga, no strict Gladbach filter)
|
||||
all_results.extend(generic_scrape(
|
||||
'sportschau.de',
|
||||
['https://www.sportschau.de/fussball/bundesliga/'],
|
||||
url_filter=lambda u: 'sportschau.de' in u and '/fussball/' in u,
|
||||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ss]portschau.*$', '', t).strip(),
|
||||
gladbach_check=False,
|
||||
))
|
||||
|
||||
# 6. fussballtransfers.com
|
||||
all_results.extend(generic_scrape(
|
||||
'fussballtransfers.com',
|
||||
['https://www.fussballtransfers.com/bundesliga', 'https://www.fussballtransfers.com/'],
|
||||
url_filter=lambda u: 'fussballtransfers.com' in u,
|
||||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ff]ussball\s*[Tt]ransfers.*$', '', t).strip(),
|
||||
))
|
||||
|
||||
# Deduplicate by URL
|
||||
seen_urls = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
if r['url'] not in seen_urls:
|
||||
seen_urls.add(r['url'])
|
||||
deduped.append(r)
|
||||
|
||||
# Prioritize: Gladbach articles first (with 2+ keyword hits), then the rest
|
||||
gladbach_heavy = [r for r in deduped if has_gladbach_keyword(r['title'] + ' ' + r['full_text'], min_hits=2)]
|
||||
gladbach_light = [r for r in deduped if r not in gladbach_heavy and has_gladbach_keyword(r['title'] + ' ' + r['full_text'], min_hits=1)]
|
||||
other = [r for r in deduped if r not in gladbach_heavy and r not in gladbach_light]
|
||||
|
||||
# Take 5: prefer heavy Gladbach, then light Gladbach, then other
|
||||
final = gladbach_heavy + gladbach_light + other
|
||||
final = final[:5]
|
||||
|
||||
print("\n\n" + "=" * 60)
|
||||
print(f"FINAL RESULTS: {len(final)} articles")
|
||||
print("=" * 60)
|
||||
|
||||
for i, r in enumerate(final, 1):
|
||||
print(f"\n--- Artikel {i} ---")
|
||||
print(f"Quelle: {r['source']}")
|
||||
print(f"Titel: {r['title']}")
|
||||
print(f"Datum: {r['date']}")
|
||||
print(f"URL: {r['url']}")
|
||||
print(f"Textlänge: {len(r['full_text'])} Zeichen")
|
||||
|
||||
with open('/home/hermes/workspace/gladbach_news.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(final, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"\nResults saved to /home/hermes/workspace/gladbach_news.json")
|
||||
return final
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user