Hermes Agent
85 lines
3.6 KiB
Python
85 lines
3.6 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
import json
|
|
import sys
|
|
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
|
}
|
|
|
|
def fetch_url(url, timeout=20):
|
|
try:
|
|
resp = requests.get(url, headers=HEADERS, timeout=timeout)
|
|
resp.encoding = resp.apparent_encoding or 'utf-8'
|
|
return resp.text
|
|
except Exception as e:
|
|
print(f"ERROR: {url}: {e}", file=sys.stderr)
|
|
return None
|
|
|
|
def get_fulltext(html):
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for selector in ["article", ".article-content", ".post-content", ".entry-content",
|
|
".news-content", ".article-body", ".story-body", "main", ".content", "#content"]:
|
|
content_el = soup.select_one(selector)
|
|
if content_el:
|
|
for unwanted in content_el.select("script, style, nav, .ad, .advertisement, .comments, .related"):
|
|
unwanted.decompose()
|
|
text = content_el.get_text(separator="\n", strip=True)
|
|
if len(text) > 200:
|
|
return text[:5000]
|
|
paragraphs = soup.select("article p, .content p, main p")
|
|
if paragraphs:
|
|
return "\n".join(p.get_text(strip=True) for p in paragraphs[:20])[:5000]
|
|
return ""
|
|
|
|
# Fetch remaining gladbachlive articles
|
|
urls = [
|
|
("Wird der WM-Traum wahr? Gladbach-Star vor Duell mit Fohlen-Kollegen",
|
|
"https://www.gladbachlive.de/news/wird-der-wm-traum-wahr-gladbach-star-vor-duell-mit-fohlen-kollegen-1296117"),
|
|
("Große Begehrlichkeit geweckt: Borussia-Talent vor dem Absprung?",
|
|
"https://www.gladbachlive.de/news/grosse-begehrlichkeit-geweckt-borussia-talent-vor-dem-absprung-1295909"),
|
|
("Gladbacher vor der WM: Elvedi-Truppe patzt",
|
|
"https://www.gladbachlive.de/news/gladbacher-vor-der-wm-elvedi-truppe-patzt-gegen-aussenseiter-1295352"),
|
|
("Fohlenelf vor machbarer Aufgabe im DFB-Pokal",
|
|
"https://www.gladbachlive.de/news/in-bundesliga-stadion-gladbach-vor-machbarer-aufgabe-im-dfb-pokal-1295280"),
|
|
("Hüne aus Belgien als Verstärkung für die Abwehr?",
|
|
"https://www.gladbachlive.de/news/geruechte-in-gladbach-abwehr-huene-als-defensiv-verstaerkung-1295178"),
|
|
]
|
|
|
|
results = {}
|
|
for title, url in urls:
|
|
print(f"Fetching: {url}", file=sys.stderr)
|
|
html = fetch_url(url)
|
|
if html:
|
|
ft = get_fulltext(html)
|
|
results[url] = {"title": title, "fulltext": ft, "length": len(ft)}
|
|
print(f" Got {len(ft)} chars", file=sys.stderr)
|
|
|
|
# Also get TorFabrik article
|
|
torfabrik_url = "https://www.torfabrik.de/"
|
|
html = fetch_url(torfabrik_url)
|
|
if html:
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# Find the top article link
|
|
top_news = soup.select_one(".top-news a") or soup.select_one("h1 a, .news-header a")
|
|
if top_news and top_news.get("href"):
|
|
article_url = "https://www.torfabrik.de" + top_news["href"] if not top_news["href"].startswith("http") else top_news["href"]
|
|
print(f"Fetching TorFabrik article: {article_url}", file=sys.stderr)
|
|
article_html = fetch_url(article_url)
|
|
if article_html:
|
|
ft = get_fulltext(article_html)
|
|
results[article_url] = {"title": "Borussia verpflichtet Isac Lidberg", "fulltext": ft, "length": len(ft)}
|
|
print(f" Got {len(ft)} chars", file=sys.stderr)
|
|
|
|
# Save results
|
|
with open("/home/hermes/workspace/more_fulltext.json", "w") as f:
|
|
json.dump(results, f, ensure_ascii=False, indent=2)
|
|
|
|
# Print summary
|
|
for url, data in results.items():
|
|
print(f"\n{'='*60}")
|
|
print(f"TITLE: {data['title']}")
|
|
print(f"URL: {url}")
|
|
print(f"LENGTH: {data['length']} chars")
|
|
print(f"TEXT:\n{data['fulltext'][:1000]}")
|