fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)

2026-06-23 23:47:51 +02:00
parent e73e0e88de
commit 4d90e4249d
656 changed files with 2602398 additions and 3 deletions
@@ -0,0 +1,84 @@
+import requests
+from bs4 import BeautifulSoup
+import json
+import sys
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+}
+
+def fetch_url(url, timeout=20):
+    try:
+        resp = requests.get(url, headers=HEADERS, timeout=timeout)
+        resp.encoding = resp.apparent_encoding or 'utf-8'
+        return resp.text
+    except Exception as e:
+        print(f"ERROR: {url}: {e}", file=sys.stderr)
+        return None
+
+def get_fulltext(html):
+    soup = BeautifulSoup(html, 'html.parser')
+    for selector in ["article", ".article-content", ".post-content", ".entry-content",
+                      ".news-content", ".article-body", ".story-body", "main", ".content", "#content"]:
+        content_el = soup.select_one(selector)
+        if content_el:
+            for unwanted in content_el.select("script, style, nav, .ad, .advertisement, .comments, .related"):
+                unwanted.decompose()
+            text = content_el.get_text(separator="\n", strip=True)
+            if len(text) > 200:
+                return text[:5000]
+    paragraphs = soup.select("article p, .content p, main p")
+    if paragraphs:
+        return "\n".join(p.get_text(strip=True) for p in paragraphs[:20])[:5000]
+    return ""
+
+# Fetch remaining gladbachlive articles
+urls = [
+    ("Wird der WM-Traum wahr? Gladbach-Star vor Duell mit Fohlen-Kollegen",
+     "https://www.gladbachlive.de/news/wird-der-wm-traum-wahr-gladbach-star-vor-duell-mit-fohlen-kollegen-1296117"),
+    ("Große Begehrlichkeit geweckt: Borussia-Talent vor dem Absprung?",
+     "https://www.gladbachlive.de/news/grosse-begehrlichkeit-geweckt-borussia-talent-vor-dem-absprung-1295909"),
+    ("Gladbacher vor der WM: Elvedi-Truppe patzt",
+     "https://www.gladbachlive.de/news/gladbacher-vor-der-wm-elvedi-truppe-patzt-gegen-aussenseiter-1295352"),
+    ("Fohlenelf vor machbarer Aufgabe im DFB-Pokal",
+     "https://www.gladbachlive.de/news/in-bundesliga-stadion-gladbach-vor-machbarer-aufgabe-im-dfb-pokal-1295280"),
+    ("Hüne aus Belgien als Verstärkung für die Abwehr?",
+     "https://www.gladbachlive.de/news/geruechte-in-gladbach-abwehr-huene-als-defensiv-verstaerkung-1295178"),
+]
+
+results = {}
+for title, url in urls:
+    print(f"Fetching: {url}", file=sys.stderr)
+    html = fetch_url(url)
+    if html:
+        ft = get_fulltext(html)
+        results[url] = {"title": title, "fulltext": ft, "length": len(ft)}
+        print(f"  Got {len(ft)} chars", file=sys.stderr)
+
+# Also get TorFabrik article
+torfabrik_url = "https://www.torfabrik.de/"
+html = fetch_url(torfabrik_url)
+if html:
+    soup = BeautifulSoup(html, 'html.parser')
+    # Find the top article link
+    top_news = soup.select_one(".top-news a") or soup.select_one("h1 a, .news-header a")
+    if top_news and top_news.get("href"):
+        article_url = "https://www.torfabrik.de" + top_news["href"] if not top_news["href"].startswith("http") else top_news["href"]
+        print(f"Fetching TorFabrik article: {article_url}", file=sys.stderr)
+        article_html = fetch_url(article_url)
+        if article_html:
+            ft = get_fulltext(article_html)
+            results[article_url] = {"title": "Borussia verpflichtet Isac Lidberg", "fulltext": ft, "length": len(ft)}
+            print(f"  Got {len(ft)} chars", file=sys.stderr)
+
+# Save results
+with open("/home/hermes/workspace/more_fulltext.json", "w") as f:
+    json.dump(results, f, ensure_ascii=False, indent=2)
+
+# Print summary
+for url, data in results.items():
+    print(f"\n{'='*60}")
+    print(f"TITLE: {data['title']}")
+    print(f"URL: {url}")
+    print(f"LENGTH: {data['length']} chars")
+    print(f"TEXT:\n{data['fulltext'][:1000]}")