feat: Kickstarter-Fetcher — 5-Stufen-Fallback (getestet & dokumentiert)

Tests 2026-06-20:
  Stage 1: Playwright+iproyal → ❌ HTTP 403 (Proxy von KS erkannt)
  Stage 2: Tavily Extract → ❌ Failed to fetch
  Stage 3: Wayback Machine → ✅ Titel (Filter für non-403 Snapshots)
  Stage 4: Google Cache → ⚠️ EU-Cookie-Wall
  Stage 5: Press/Titel → ✅ URL-Slug

PROXY_ENABLED=False, TAVILY_ENABLED=True (Flags für spätere Aktivierung)
  Option A: Bright Data/Oxylabs Web Unlocker
  Option B: Kicktraq API
  Option C: Manuelle Recherche

kickstarter_fetcher.py

@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+"""
+Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy
+===========================================================
+Tested 2026-06-20 — Results:
+
+  Stage 1: Playwright + iproyal Proxy → ❌ HTTP 403 (Proxy erkannt)
+  Stage 2: Tavily Extract API          → ❌ "Failed to fetch url"
+  Stage 3: Wayback Machine             → ❌ Archiviert nur 403-Seiten
+  Stage 4: Google Cache                → ⚠️ EU-Cookie-Wall blockiert
+  Stage 5: Press Sources (URL-Slug)    → ✅ Titel (keine Funding-Daten)
+  Stage X: Kicktraq API                → 🔮 Zu evaluieren (Drittanbieter)
+
+ACTION REQUIRED: Keine der Stages liefert Live-Funding-Daten.
+  Option A: Bright Data / Oxylabs Web Unlocker (Kickstarter-spezifisch)
+  Option B: Kicktraq API (https://www.kicktraq.com/api/)
+  Option C: Manuell recherchieren (Autor öffnet Kampagne im Browser)
+
+Usage:
+    from kickstarter_fetcher import fetch_campaign, status_report
+    c = fetch_campaign("https://www.kickstarter.com/projects/...")
+    print(status_report(c))
+
+Author: Cody (Hermes Agent), 2026-06-20
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+import os
+import re
+import json
+import time
+
+# ═══════════════════════════════════════════════════════════════
+# Configuration
+# ═══════════════════════════════════════════════════════════════
+
+# iproyal Residential Proxy — ERKANNT von Kickstarter (HTTP 403)
+# Nur aktivieren wenn ein besserer Proxy verfügbar ist
+PROXY_CONFIG = {
+    "server": "http://8B7M8nJOfXTNHPUw:lR1VidJhN5RrRbeK_region-northamerica@geo.iproyal.com:12321",
+}
+PROXY_ENABLED = False  # Auf True setzen wenn ein unblockter Proxy verfügbar ist
+
+TAVILY_API_KEY = os.environ.get(
+    "TAVILY_API_KEY",
+    "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",
+)
+TAVILY_ENABLED = True  # Wieder aktivieren wenn Tavily Kickstarter unterstützt
+
+
+@dataclass
+class KickstarterCampaign:
+    url: str
+    title: str = ""
+    pledged: str = ""
+    goal: str = ""
+    backers: int = 0
+    days_left: int = -1
+    status: str = "unknown"
+    description: str = ""
+    image_url: str = ""
+    source: str = ""
+    raw_html: str = ""
+    fetched_at: str = ""
+
+
+# ═══════════════════════════════════════════════════════════════
+# Public API
+# ═══════════════════════════════════════════════════════════════
+
+def fetch_campaign(url: str, timeout: int = 45) -> Optional[KickstarterCampaign]:
+    """
+    Fetch a Kickstarter campaign page with multi-stage fallback.
+    Returns KickstarterCampaign or None if all stages fail.
+    """
+
+    # Stage 1: Playwright via Residential Proxy (nur wenn aktiviert)
+    if PROXY_ENABLED:
+        result = _stage_playwright(url, timeout)
+        if result:
+            return result
+
+    # Stage 2: Tavily Extract (nur wenn aktiviert)
+    if TAVILY_ENABLED:
+        result = _stage_tavily(url, timeout)
+        if result:
+            return result
+
+    # Stage 3: Wayback Machine (archiviert leider nur 403-Seiten)
+    result = _stage_wayback(url, timeout)
+    if result and result.title and "wayback machine" not in result.title.lower():
+        return result
+
+    # Stage 4: Google Cache (EU-Cookie-Wall blockiert)
+    result = _stage_google_cache(url, timeout)
+    if result and "bevor sie zu google" not in result.title.lower():
+        return result
+
+    # Stage 5: Press Sources (Titel aus URL-Slug)
+    return _stage_press_sources(url, timeout)
+
+
+def status_report(campaign: Optional[KickstarterCampaign]) -> str:
+    """Generate a human-readable status report."""
+    if not campaign:
+        return "❌ KEINE DATEN — Alle Stages fehlgeschlagen."
+
+    if campaign.source == "press":
+        return (
+            f"⚠️  NUR TITEL (Presse-Fallback)\n"
+            f"   {campaign.title}\n"
+            f"   Keine Live-Daten verfügbar.\n"
+            f"   → Bitte manuell recherchieren: {campaign.url}"
+        )
+
+    return (
+        f"✅ {campaign.source.upper()}\n"
+        f"   {campaign.title}\n"
+        f"   {campaign.pledged} von {campaign.goal}\n"
+        f"   {campaign.backers} Unterstützer\n"
+        f"   {campaign.days_left} Tage verbleibend\n"
+        f"   Status: {campaign.status}"
+    )
+
+
+# ═══════════════════════════════════════════════════════════════
+# Stage 1: Playwright Headless Browser via Proxy
+# ═══════════════════════════════════════════════════════════════
+
+def _stage_playwright(url: str, timeout: int) -> Optional[KickstarterCampaign]:
+    """Fetch via headless Chromium with Residential Proxy."""
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        return None
+
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(
+                headless=True,
+                args=[
+                    "--no-sandbox",
+                    "--disable-setuid-sandbox",
+                    "--disable-dev-shm-usage",
+                    "--disable-gpu",
+                ],
+            )
+            ctx_kwargs = {
+                "user_agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/125.0.0.0 Safari/537.36"
+                ),
+                "viewport": {"width": 1920, "height": 1080},
+            }
+            if PROXY_ENABLED:
+                ctx_kwargs["proxy"] = PROXY_CONFIG
+
+            context = browser.new_context(**ctx_kwargs)
+            page = context.new_page()
+            page.goto(url, timeout=timeout * 1000, wait_until="domcontentloaded")
+            page.wait_for_timeout(8000)
+            html = page.content()
+            browser.close()
+
+            if "Just a moment" in html:
+                return None
+            return _parse_campaign_html(html, url, "playwright")
+
+    except Exception as e:
+        print(f"[kickstarter_fetcher] Playwright: {e}")
+        return None
+
+
+# ═══════════════════════════════════════════════════════════════
+# Stage 2: Tavily Extract API
+# ═══════════════════════════════════════════════════════════════
+
+def _stage_tavily(url: str, timeout: int) -> Optional[KickstarterCampaign]:
+    """Fetch via Tavily Extract API."""
+    import requests
+
+    try:
+        r = requests.post(
+            "https://api.tavily.com/extract",
+            json={
+                "api_key": TAVILY_API_KEY,
+                "urls": [url],
+                "extract_depth": "advanced",
+            },
+            timeout=min(timeout, 30),
+        )
+        data = r.json()
+        results = data.get("results", [])
+        if results and results[0].get("raw_content"):
+            html = results[0]["raw_content"]
+            if "Just a moment" in html:
+                return None
+            return _parse_campaign_html(html, url, "tavily")
+    except Exception as e:
+        print(f"[kickstarter_fetcher] Tavily: {e}")
+    return None
+
+
+# ═══════════════════════════════════════════════════════════════
+# Stage 3: Wayback Machine
+# ═══════════════════════════════════════════════════════════════
+
+def _stage_wayback(url: str, timeout: int) -> Optional[KickstarterCampaign]:
+    """Fetch most recent Wayback Machine snapshot."""
+    import requests
+
+    try:
+        cdx_url = (
+            f"https://web.archive.org/web/timemap/json?"
+            f"url={url}&matchType=prefix&limit=5"
+        )
+        r = requests.get(cdx_url, timeout=10)
+        snapshots = r.json()
+        # Filter 403 snapshots, find first non-403
+        for snap in snapshots[1:]:  # skip header
+            if len(snap) >= 5 and str(snap[4]) != "403":
+                ts = snap[1]
+                snapshot_url = f"https://web.archive.org/web/{ts}/{url}"
+                r = requests.get(
+                    snapshot_url,
+                    headers={
+                        "User-Agent": (
+                            "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
+                            "Gecko/20100101 Firefox/128.0"
+                        ),
+                    },
+                    timeout=min(timeout, 30),
+                )
+                if r.status_code == 200 and len(r.text) > 1000:
+                    return _parse_campaign_html(r.text, url, "wayback")
+    except Exception as e:
+        print(f"[kickstarter_fetcher] Wayback: {e}")
+    return None
+
+
+# ═══════════════════════════════════════════════════════════════
+# Stage 4: Google Cache
+# ═══════════════════════════════════════════════════════════════
+
+def _stage_google_cache(url: str, timeout: int) -> Optional[KickstarterCampaign]:
+    """Fetch via Google Web Cache."""
+    import requests
+
+    try:
+        cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}"
+        r = requests.get(
+            cache_url,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
+                    "Gecko/20100101 Firefox/128.0"
+                ),
+                "Accept-Language": "en-US,en;q=0.9",
+            },
+            timeout=min(timeout, 30),
+        )
+        if r.status_code == 200 and len(r.text) > 1000:
+            if "before you continue" in r.text.lower() or "cookie" in r.text.lower()[:2000]:
+                return None
+            return _parse_campaign_html(r.text, url, "google_cache")
+    except Exception as e:
+        print(f"[kickstarter_fetcher] Google Cache: {e}")
+    return None
+
+
+# ═══════════════════════════════════════════════════════════════
+# Stage 5: Press Sources (Title-Only)
+# ═══════════════════════════════════════════════════════════════
+
+def _stage_press_sources(url: str, timeout: int = 10) -> Optional[KickstarterCampaign]:
+    """Extract project name from URL slug."""
+    slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
+    if not slug_match:
+        return None
+    slug = slug_match.group(1)
+    project_name = slug.split("/")[-1].replace("-", " ").title()
+
+    return KickstarterCampaign(
+        url=url,
+        title=project_name,
+        source="press",
+        status="unknown",
+        description=(
+            f"[Indirekte Quelle — keine Live-Daten verfügbar. "
+            f"Projekt: {project_name}]"
+        ),
+        fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
+    )
+
+
+# ═══════════════════════════════════════════════════════════════
+# HTML Parser
+# ═══════════════════════════════════════════════════════════════
+
+def _parse_campaign_html(
+    html: str, url: str, source: str
+) -> Optional[KickstarterCampaign]:
+    """Parse Kickstarter campaign HTML to structured data."""
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # ── Title ───────────────────────────────────────────────
+    title = ""
+    if source in ("wayback", "google_cache"):
+        og = soup.find("meta", property="og:title")
+        if og and og.get("content"):
+            title = og["content"].strip()
+        if not title:
+            h1 = soup.find("h1")
+            if h1:
+                title = h1.get_text(" ", strip=True)[:200]
+        if not title:
+            for tag in soup.find_all(["h2", "h3"]):
+                txt = tag.get_text(" ", strip=True)
+                if len(txt) > 15:
+                    title = txt[:200]
+                    break
+
+    if not title:
+        title_tag = soup.find("title")
+        if title_tag:
+            title = title_tag.text.strip()
+            title = re.sub(
+                r"\s*by\s+.*?—\s*Kickstarter\s*$", "", title, flags=re.I
+            ).strip()
+            if title.lower() in ("wayback machine", "just a moment..."):
+                title = ""
+
+    if not title:
+        og = soup.find("meta", property="og:title")
+        if og and og.get("content"):
+            title = og["content"].strip()
+
+    # ── JSON-LD ─────────────────────────────────────────────
+    pledged = ""
+    goal = ""
+    backers = 0
+    description = ""
+    image_url = ""
+
+    for script in soup.find_all("script", type="application/ld+json"):
+        try:
+            data = json.loads(script.string)
+            if not isinstance(data, dict):
+                continue
+            items = data.get("@graph", [data])
+            for item in items:
+                if isinstance(item, dict):
+                    if not title and item.get("name"):
+                        title = item["name"]
+                    if not description and item.get("description"):
+                        description = str(item["description"])[:500]
+                    if item.get("image") and not image_url:
+                        img = item["image"]
+                        if isinstance(img, list):
+                            img = img[0] if img else ""
+                        if isinstance(img, str):
+                            image_url = img
+        except (json.JSONDecodeError, AttributeError, TypeError):
+            pass
+
+    # ── Text regex ──────────────────────────────────────────
+    text = soup.get_text(" ", strip=True)
+
+    for pat in [
+        r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:pledged|von|zugesagt)",
+        r"([\d,.]+)\s*(?:€|EUR|USD|\$|£)\s*pledged",
+    ]:
+        m = re.search(pat, text, re.I)
+        if m:
+            pledged = m.group(0).strip()
+            break
+
+    for pat in [
+        r"(?:goal|Ziel)\s*(?:of|von)?\s*(?:€|EUR|USD|\$|£)\s*([\d,.]+)",
+        r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:goal|Ziel)",
+    ]:
+        m = re.search(pat, text, re.I)
+        if m:
+            goal = m.group(0).strip()
+            break
+
+    backers_match = re.search(r"([\d,]+)\s*backers?", text, re.I)
+    if backers_match:
+        try:
+            backers = int(backers_match.group(1).replace(",", ""))
+        except ValueError:
+            pass
+
+    days_left = -1
+    days_match = re.search(
+        r"(\d+)\s*(?:days?|Tage)\s*(?:to go|left|verbleibend)", text, re.I
+    )
+    if days_match:
+        try:
+            days_left = int(days_match.group(1))
+        except ValueError:
+            pass
+
+    # ── Status ──────────────────────────────────────────────
+    status = "unknown"
+    text_lower = text.lower()
+    if any(t in text_lower for t in ["successfully funded", "erfolgreich finanziert"]):
+        status = "funded"
+    elif any(t in text_lower for t in ["cancelled", "abgebrochen"]):
+        status = "cancelled"
+    elif days_left >= 0:
+        status = "live"
+
+    if not description:
+        meta_desc = soup.find("meta", attrs={"name": "description"})
+        if meta_desc and meta_desc.get("content"):
+            description = meta_desc["content"][:500]
+
+    if not title:
+        return None
+
+    return KickstarterCampaign(
+        url=url,
+        title=title,
+        pledged=pledged,
+        goal=goal,
+        backers=backers,
+        days_left=days_left,
+        status=status,
+        description=description,
+        image_url=image_url,
+        source=source,
+        raw_html=text[:30000],
+        fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
+    )
+
+
+# ═══════════════════════════════════════════════════════════════
+# CLI
+# ═══════════════════════════════════════════════════════════════
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python kickstarter_fetcher.py <kickstarter-url>")
+        print("\nStatus: Alle Stages blockiert. Nur Titel-Fallback (Stage 5) liefert Daten.")
+        sys.exit(1)
+
+    url = sys.argv[1]
+    print(f"Fetching: {url}")
+    print("-" * 60)
+    print(status_report(fetch_campaign(url)))