Hermes Agent
302 lines
11 KiB
Python
302 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Scrape Gladbach-focused football news from German sources."""
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
import json
|
||
import time
|
||
import re
|
||
|
||
HEADERS = {
|
||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||
'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
|
||
}
|
||
|
||
GLADBACH_KW = [
|
||
'gladbach', 'borussia', 'mönchengladbach', 'fohlen', 'fohlenelf',
|
||
'borussia park', 'niederrhein', 'borussen',
|
||
'elvedi', 'plea', 'honorat', 'neuhaus', 'weigl', 'koné',
|
||
'scally', 'oultara', 'friedrich', 'itakura', 'ngoumou',
|
||
'cvančara', 'civancara', 'ranos', 'hack',
|
||
'reitz', 'netz', 'wöber', 'woeber', 'kleindienst',
|
||
'seoane', 'virkus',
|
||
]
|
||
|
||
def fetch_url(url, retries=2):
|
||
for attempt in range(retries + 1):
|
||
try:
|
||
resp = requests.get(url, headers=HEADERS, timeout=20)
|
||
resp.raise_for_status()
|
||
resp.encoding = resp.apparent_encoding or 'utf-8'
|
||
return resp
|
||
except Exception as e:
|
||
if attempt < retries:
|
||
time.sleep(2)
|
||
else:
|
||
print(f" FAILED: {url} — {e}")
|
||
return None
|
||
|
||
def clean_text(text):
|
||
return re.sub(r'\s+', ' ', text).strip()
|
||
|
||
def is_recent_date(date_str):
|
||
patterns = [
|
||
r'01\.06\.2026', r'02\.06\.2026',
|
||
r'1\.\s*Juni\s*2026', r'2\.\s*Juni\s*2026',
|
||
r'01\.06\.26', r'02\.06\.26',
|
||
r'2026-06-01', r'2026-06-02',
|
||
r'vor \d+ Stunden', r'vor \d+ Minuten',
|
||
]
|
||
for pat in patterns:
|
||
if re.search(pat, date_str, re.IGNORECASE):
|
||
return True
|
||
return False
|
||
|
||
def has_gladbach_keyword(text, min_hits=1):
|
||
text_lower = text.lower()
|
||
hits = sum(1 for kw in GLADBACH_KW if kw in text_lower)
|
||
return hits >= min_hits
|
||
|
||
def extract_article_text(soup):
|
||
selectors = [
|
||
'article', '.article-content', '.article-body', '.post-content',
|
||
'.entry-content', '.content', '.story-body', '.article__body',
|
||
'.article-text', '.news-text', 'main', '.main-content',
|
||
'[itemprop="articleBody"]', '.article__content', '.post__content',
|
||
'.single-content', '#article-content',
|
||
]
|
||
for sel in selectors:
|
||
elem = soup.select_one(sel)
|
||
if elem:
|
||
for unwanted in elem.find_all(['script', 'style', 'nav', 'aside', 'footer', 'header', 'iframe', 'form']):
|
||
unwanted.decompose()
|
||
text = elem.get_text(separator=' ', strip=True)
|
||
text = clean_text(text)
|
||
if len(text) > 200:
|
||
return text
|
||
paragraphs = soup.find_all('p')
|
||
texts = [clean_text(p.get_text()) for p in paragraphs if len(p.get_text().strip()) > 50]
|
||
if texts:
|
||
return ' '.join(texts)
|
||
body = soup.find('body')
|
||
if body:
|
||
for unwanted in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'iframe']):
|
||
unwanted.decompose()
|
||
return clean_text(body.get_text(separator=' ', strip=True))
|
||
return ""
|
||
|
||
def extract_date(soup):
|
||
date_patterns = [
|
||
('meta[property="article:published_time"]', 'content'),
|
||
('meta[name="date"]', 'content'),
|
||
('meta[name="pubdate"]', 'content'),
|
||
('time', 'datetime'),
|
||
('time[datetime]', 'datetime'),
|
||
]
|
||
for selector, attr in date_patterns:
|
||
elem = soup.select_one(selector)
|
||
if elem:
|
||
val = elem.get(attr, '') if attr else elem.get_text(strip=True)
|
||
if val and len(val) > 5:
|
||
return val
|
||
text = soup.get_text()
|
||
m = re.search(r'(\d{1,2}\.\s*\d{1,2}\.\s*(?:20)?2[456])', text)
|
||
return m.group(1) if m else ""
|
||
|
||
def generic_scrape(name, base_urls, url_filter=None, title_cleanup=None, gladbach_check=True):
|
||
"""Generic scraper for a news site."""
|
||
print(f"\n{'='*60}\nSCRAPING: {name}\n{'='*60}")
|
||
results = []
|
||
|
||
for base_url in base_urls:
|
||
print(f" Trying: {base_url}")
|
||
resp = fetch_url(base_url)
|
||
if not resp:
|
||
continue
|
||
|
||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||
links = soup.find_all('a', href=True)
|
||
article_urls = set()
|
||
|
||
for link in links:
|
||
href = link['href']
|
||
if not href or href.startswith('#') or href.startswith('javascript:') or href.startswith('mailto:'):
|
||
continue
|
||
if href.endswith(('.jpg', '.png', '.gif', '.pdf', '.css', '.js', '.ico')):
|
||
continue
|
||
|
||
# Build full URL
|
||
if href.startswith('http'):
|
||
full_url = href
|
||
elif href.startswith('//'):
|
||
full_url = 'https:' + href
|
||
elif href.startswith('/'):
|
||
# Extract domain from base_url
|
||
domain = re.match(r'https?://[^/]+', base_url)
|
||
if domain:
|
||
full_url = domain.group(0) + href
|
||
else:
|
||
continue
|
||
else:
|
||
continue
|
||
|
||
if full_url in article_urls:
|
||
continue
|
||
|
||
# Apply filter if provided
|
||
if url_filter:
|
||
if url_filter(full_url):
|
||
article_urls.add(full_url)
|
||
else:
|
||
article_urls.add(full_url)
|
||
|
||
print(f" Found {len(article_urls)} potential article URLs (sampling first 30)")
|
||
|
||
for art_url in list(article_urls)[:30]:
|
||
print(f" Fetching: {art_url}")
|
||
art_resp = fetch_url(art_url)
|
||
if not art_resp:
|
||
continue
|
||
|
||
art_soup = BeautifulSoup(art_resp.text, 'html.parser')
|
||
|
||
# Extract title
|
||
title = None
|
||
title_elem = art_soup.find('title')
|
||
if title_elem:
|
||
title = clean_text(title_elem.get_text())
|
||
if title_cleanup:
|
||
title = title_cleanup(title)
|
||
if not title or len(title) < 15:
|
||
h1 = art_soup.find('h1')
|
||
if h1:
|
||
title = clean_text(h1.get_text())
|
||
if not title or len(title) < 15:
|
||
continue
|
||
|
||
date_str = extract_date(art_soup)
|
||
full_text = extract_article_text(art_soup)
|
||
|
||
if not full_text or len(full_text) < 100:
|
||
print(f" Skipping (no usable text)")
|
||
continue
|
||
|
||
if date_str and not is_recent_date(date_str):
|
||
print(f" Skipping (date: {date_str})")
|
||
continue
|
||
|
||
if gladbach_check:
|
||
combined = (title + ' ' + full_text).lower()
|
||
if not has_gladbach_keyword(combined, min_hits=1):
|
||
print(f" Not Gladbach-related, skipping")
|
||
continue
|
||
|
||
print(f" ✓ FOUND: {title[:80]}")
|
||
results.append({
|
||
'source': name,
|
||
'title': title,
|
||
'url': art_url,
|
||
'date': date_str or 'Kein Datum gefunden',
|
||
'full_text': full_text[:3000],
|
||
})
|
||
|
||
if len(results) >= 2:
|
||
break
|
||
|
||
if results:
|
||
break
|
||
|
||
return results
|
||
|
||
def main():
|
||
all_results = []
|
||
|
||
# 1. torfabrik.de
|
||
all_results.extend(generic_scrape(
|
||
'torfabrik.de',
|
||
['https://www.torfabrik.de/', 'https://www.torfabrik.de/borussia-moenchengladbach/'],
|
||
url_filter=lambda u: 'torfabrik.de' in u,
|
||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Tt]orfabrik.*$', '', t).strip(),
|
||
))
|
||
|
||
# 2. fohlen-hautnah.de
|
||
all_results.extend(generic_scrape(
|
||
'fohlen-hautnah.de',
|
||
['https://www.fohlen-hautnah.de/'],
|
||
url_filter=lambda u: 'fohlen-hautnah.de' in u,
|
||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ff]ohlen[-\s]*[Hh]autnah.*$', '', t).strip(),
|
||
gladbach_check=False, # All articles are Gladbach-related
|
||
))
|
||
|
||
# 3. gladbachlive.de
|
||
all_results.extend(generic_scrape(
|
||
'gladbachlive.de',
|
||
['https://www.gladbachlive.de/', 'https://www.gladbachlive.de/news/'],
|
||
url_filter=lambda u: 'gladbachlive.de' in u,
|
||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Gg]ladbach[Ll]ive.*$', '', t).strip(),
|
||
))
|
||
|
||
# 4. kicker.de
|
||
all_results.extend(generic_scrape(
|
||
'kicker.de',
|
||
['https://www.kicker.de/borussia-moenchengladbach/news', 'https://www.kicker.de/bundesliga/aktuell'],
|
||
url_filter=lambda u: 'kicker.de' in u and any(kw in u.lower() for kw in ['gladbach', 'borussia-mg', 'bundesliga', 'news', 'transfer']),
|
||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Kk]icker.*$', '', t).strip(),
|
||
))
|
||
|
||
# 5. sportschau.de (general Bundesliga, no strict Gladbach filter)
|
||
all_results.extend(generic_scrape(
|
||
'sportschau.de',
|
||
['https://www.sportschau.de/fussball/bundesliga/'],
|
||
url_filter=lambda u: 'sportschau.de' in u and '/fussball/' in u,
|
||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ss]portschau.*$', '', t).strip(),
|
||
gladbach_check=False,
|
||
))
|
||
|
||
# 6. fussballtransfers.com
|
||
all_results.extend(generic_scrape(
|
||
'fussballtransfers.com',
|
||
['https://www.fussballtransfers.com/bundesliga', 'https://www.fussballtransfers.com/'],
|
||
url_filter=lambda u: 'fussballtransfers.com' in u,
|
||
title_cleanup=lambda t: re.sub(r'\s*[-–|]\s*[Ff]ussball\s*[Tt]ransfers.*$', '', t).strip(),
|
||
))
|
||
|
||
# Deduplicate by URL
|
||
seen_urls = set()
|
||
deduped = []
|
||
for r in all_results:
|
||
if r['url'] not in seen_urls:
|
||
seen_urls.add(r['url'])
|
||
deduped.append(r)
|
||
|
||
# Prioritize: Gladbach articles first (with 2+ keyword hits), then the rest
|
||
gladbach_heavy = [r for r in deduped if has_gladbach_keyword(r['title'] + ' ' + r['full_text'], min_hits=2)]
|
||
gladbach_light = [r for r in deduped if r not in gladbach_heavy and has_gladbach_keyword(r['title'] + ' ' + r['full_text'], min_hits=1)]
|
||
other = [r for r in deduped if r not in gladbach_heavy and r not in gladbach_light]
|
||
|
||
# Take 5: prefer heavy Gladbach, then light Gladbach, then other
|
||
final = gladbach_heavy + gladbach_light + other
|
||
final = final[:5]
|
||
|
||
print("\n\n" + "=" * 60)
|
||
print(f"FINAL RESULTS: {len(final)} articles")
|
||
print("=" * 60)
|
||
|
||
for i, r in enumerate(final, 1):
|
||
print(f"\n--- Artikel {i} ---")
|
||
print(f"Quelle: {r['source']}")
|
||
print(f"Titel: {r['title']}")
|
||
print(f"Datum: {r['date']}")
|
||
print(f"URL: {r['url']}")
|
||
print(f"Textlänge: {len(r['full_text'])} Zeichen")
|
||
|
||
with open('/home/hermes/workspace/gladbach_news.json', 'w', encoding='utf-8') as f:
|
||
json.dump(final, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"\nResults saved to /home/hermes/workspace/gladbach_news.json")
|
||
return final
|
||
|
||
if __name__ == '__main__':
|
||
main()
|