feat: Kickstarter LIVE-Daten via FlareSolverr — DURCHBRUCH!

Stage 0: FlareSolverr (Headless Chromium, localhost:8191)
  - Löst Cloudflare-Challenge automatisch
  - Extrahiert Campaign-Daten aus embedded Flat-JSON:
    project_name, project_backers_count, project_pledged_usd,
    project_goal_usd, project_state, project_deadline, project_blurb

Getestet: Earthborne Trailblazer → 49,130 / 3,969 backers / successful
  Kein Proxy nötig — FlareSolverr's Chromium reicht!

Setup: FlareSolverr in ~/workspace/FlareSolverr/
  Start: cd src && python start_flaresolverr.py
  Patch: --headless=new statt Xvfb (utils.py)

Kickstarter-Fetcher jetzt: 2 Stages (FlareSolverr → Slug-Fallback)

kickstarter_fetcher.py

@@ -2,66 +2,52 @@
 """
 Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy
 ===========================================================
-Tested 2026-06-20 — Results:
+Final version 2026-06-21 — FlareSolverr breakthrough!
 
-  Stage 1: Playwright + iproyal Proxy → ❌ HTTP 403 (Proxy erkannt)
+Stages:
-  Stage 2: Tavily Extract API          → ❌ "Failed to fetch url"
+  0. FlareSolverr (localhost:8191) → LIVE campaign data! ✅
-  Stage 3: Wayback Machine             → ❌ Archiviert nur 403-Seiten
+  1. Fallback: URL-Slug → Titel (immer)
-  Stage 4: Google Cache                → ⚠️ EU-Cookie-Wall blockiert
-  Stage 5: Press Sources (URL-Slug)    → ✅ Titel (keine Funding-Daten)
-  Stage X: Kicktraq API                → 🔮 Zu evaluieren (Drittanbieter)
 
-ACTION REQUIRED: Keine der Stages liefert Live-Funding-Daten.
+How it works:
-  Option A: Bright Data / Oxylabs Web Unlocker (Kickstarter-spezifisch)
+  FlareSolverr runs a headless Chromium that solves Cloudflare challenges.
-  Option B: Kicktraq API (https://www.kicktraq.com/api/)
+  Kickstarter embeds campaign data as flat JSON fields in the HTML:
-  Option C: Manuell recherchieren (Autor öffnet Kampagne im Browser)
+    project_name, project_backers_count, project_current_amount_pledged_usd, etc.
+  We extract these via regex → structured KickstarterCampaign object.
 
 Usage:
     from kickstarter_fetcher import fetch_campaign, status_report
     c = fetch_campaign("https://www.kickstarter.com/projects/...")
     print(status_report(c))
 
-Author: Cody (Hermes Agent), 2026-06-20
+Author: Cody (Hermes Agent), 2026-06-20/21
 """
 
 from dataclasses import dataclass
 from typing import Optional
-import os
-import re
 import json
+import re
 import time
 
+
 # ═══════════════════════════════════════════════════════════════
 # Configuration
 # ═══════════════════════════════════════════════════════════════
 
-# iproyal Residential Proxy — ERKANNT von Kickstarter (HTTP 403)
+FLARESOLVERR_URL = "http://localhost:8191/v1"
-# Nur aktivieren wenn ein besserer Proxy verfügbar ist
+FLARESOLVERR_TIMEOUT = 60  # seconds for KS page load
-PROXY_CONFIG = {
-    "server": "http://8B7M8nJOfXTNHPUw:lR1VidJhN5RrRbeK_region-northamerica@geo.iproyal.com:12321",
-}
-PROXY_ENABLED = False  # Auf True setzen wenn ein unblockter Proxy verfügbar ist
-
-TAVILY_API_KEY = os.environ.get(
-    "TAVILY_API_KEY",
-    "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",
-)
-TAVILY_ENABLED = True  # Wieder aktivieren wenn Tavily Kickstarter unterstützt
 
 
 @dataclass
 class KickstarterCampaign:
     url: str
     title: str = ""
-    pledged: str = ""
+    pledged: str = ""         # e.g. "$549,130"
-    goal: str = ""
+    goal: str = ""            # e.g. "$150,000"
     backers: int = 0
     days_left: int = -1
-    status: str = "unknown"
+    status: str = "unknown"   # "live", "successful", "cancelled"
     description: str = ""
-    image_url: str = ""
+    source: str = ""          # "flaresolverr", "slug"
-    source: str = ""
-    raw_html: str = ""
     fetched_at: str = ""
 
 
@@ -69,372 +55,175 @@ class KickstarterCampaign:
 # Public API
 # ═══════════════════════════════════════════════════════════════
 
-def fetch_campaign(url: str, timeout: int = 45) -> Optional[KickstarterCampaign]:
+def fetch_campaign(url: str, timeout: int = 90) -> Optional[KickstarterCampaign]:
     """
-    Fetch a Kickstarter campaign page with multi-stage fallback.
+    Fetch a Kickstarter campaign page.
-    Returns KickstarterCampaign or None if all stages fail.
+    Stage 0: FlareSolverr (LIVE data via headless Chromium)
+    Stage 1: URL-Slug fallback (title only)
     """
 
-    # Stage 1: Playwright via Residential Proxy (nur wenn aktiviert)
+    # Stage 0: FlareSolverr
-    if PROXY_ENABLED:
+    result = _stage_flaresolverr(url, timeout)
-        result = _stage_playwright(url, timeout)
+    if result:
-        if result:
-            return result
-
-    # Stage 2: Tavily Extract (nur wenn aktiviert)
-    if TAVILY_ENABLED:
-        result = _stage_tavily(url, timeout)
-        if result:
-            return result
-
-    # Stage 3: Wayback Machine (archiviert leider nur 403-Seiten)
-    result = _stage_wayback(url, timeout)
-    if result and result.title and "wayback machine" not in result.title.lower():
         return result
 
-    # Stage 4: Google Cache (EU-Cookie-Wall blockiert)
+    # Stage 1: Slug
-    result = _stage_google_cache(url, timeout)
+    return _stage_slug(url)
-    if result and "bevor sie zu google" not in result.title.lower():
-        return result
-
-    # Stage 5: Press Sources (Titel aus URL-Slug)
-    return _stage_press_sources(url, timeout)
 
 
 def status_report(campaign: Optional[KickstarterCampaign]) -> str:
     """Generate a human-readable status report."""
     if not campaign:
-        return "❌ KEINE DATEN — Alle Stages fehlgeschlagen."
+        return "❌ KEINE DATEN"
 
-    if campaign.source == "press":
+    if campaign.source == "slug":
         return (
-            f"⚠️  NUR TITEL (Presse-Fallback)\n"
+            f"⚠️  NUR TITEL\n"
             f"   {campaign.title}\n"
-            f"   Keine Live-Daten verfügbar.\n"
             f"   → Bitte manuell recherchieren: {campaign.url}"
         )
 
     return (
-        f"✅ {campaign.source.upper()}\n"
+        f"✅ LIVE ({campaign.source})\n"
         f"   {campaign.title}\n"
         f"   {campaign.pledged} von {campaign.goal}\n"
-        f"   {campaign.backers} Unterstützer\n"
+        f"   {campaign.backers:,} Unterstützer\n"
-        f"   {campaign.days_left} Tage verbleibend\n"
         f"   Status: {campaign.status}"
     )
 
 
 # ═══════════════════════════════════════════════════════════════
-# Stage 1: Playwright Headless Browser via Proxy
+# Stage 0: FlareSolverr → Headless Chromium → KS Page → Parse
 # ═══════════════════════════════════════════════════════════════
 
-def _stage_playwright(url: str, timeout: int) -> Optional[KickstarterCampaign]:
+def _stage_flaresolverr(url: str, timeout: int) -> Optional[KickstarterCampaign]:
-    """Fetch via headless Chromium with Residential Proxy."""
+    """
-    try:
+    Fetch Kickstarter page via FlareSolverr (headless Chromium).
-        from playwright.sync_api import sync_playwright
+    Extracts campaign data from embedded flat JSON fields.
-    except ImportError:
+    """
-        return None
-
-    try:
-        with sync_playwright() as p:
-            browser = p.chromium.launch(
-                headless=True,
-                args=[
-                    "--no-sandbox",
-                    "--disable-setuid-sandbox",
-                    "--disable-dev-shm-usage",
-                    "--disable-gpu",
-                ],
-            )
-            ctx_kwargs = {
-                "user_agent": (
-                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                    "AppleWebKit/537.36 (KHTML, like Gecko) "
-                    "Chrome/125.0.0.0 Safari/537.36"
-                ),
-                "viewport": {"width": 1920, "height": 1080},
-            }
-            if PROXY_ENABLED:
-                ctx_kwargs["proxy"] = PROXY_CONFIG
-
-            context = browser.new_context(**ctx_kwargs)
-            page = context.new_page()
-            page.goto(url, timeout=timeout * 1000, wait_until="domcontentloaded")
-            page.wait_for_timeout(8000)
-            html = page.content()
-            browser.close()
-
-            if "Just a moment" in html:
-                return None
-            return _parse_campaign_html(html, url, "playwright")
-
-    except Exception as e:
-        print(f"[kickstarter_fetcher] Playwright: {e}")
-        return None
-
-
-# ═══════════════════════════════════════════════════════════════
-# Stage 2: Tavily Extract API
-# ═══════════════════════════════════════════════════════════════
-
-def _stage_tavily(url: str, timeout: int) -> Optional[KickstarterCampaign]:
-    """Fetch via Tavily Extract API."""
     import requests
 
     try:
         r = requests.post(
-            "https://api.tavily.com/extract",
+            FLARESOLVERR_URL,
             json={
-                "api_key": TAVILY_API_KEY,
+                "cmd": "request.get",
-                "urls": [url],
+                "url": url,
-                "extract_depth": "advanced",
+                "maxTimeout": min(timeout, FLARESOLVERR_TIMEOUT) * 1000,
             },
-            timeout=min(timeout, 30),
+            timeout=timeout + 10,
         )
         data = r.json()
-        results = data.get("results", [])
-        if results and results[0].get("raw_content"):
-            html = results[0]["raw_content"]
-            if "Just a moment" in html:
-                return None
-            return _parse_campaign_html(html, url, "tavily")
     except Exception as e:
-        print(f"[kickstarter_fetcher] Tavily: {e}")
+        print(f"[kickstarter_fetcher] FlareSolverr request error: {e}")
-    return None
-
-
-# ═══════════════════════════════════════════════════════════════
-# Stage 3: Wayback Machine
-# ═══════════════════════════════════════════════════════════════
-
-def _stage_wayback(url: str, timeout: int) -> Optional[KickstarterCampaign]:
-    """Fetch most recent Wayback Machine snapshot."""
-    import requests
-
-    try:
-        cdx_url = (
-            f"https://web.archive.org/web/timemap/json?"
-            f"url={url}&matchType=prefix&limit=5"
-        )
-        r = requests.get(cdx_url, timeout=10)
-        snapshots = r.json()
-        # Filter 403 snapshots, find first non-403
-        for snap in snapshots[1:]:  # skip header
-            if len(snap) >= 5 and str(snap[4]) != "403":
-                ts = snap[1]
-                snapshot_url = f"https://web.archive.org/web/{ts}/{url}"
-                r = requests.get(
-                    snapshot_url,
-                    headers={
-                        "User-Agent": (
-                            "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
-                            "Gecko/20100101 Firefox/128.0"
-                        ),
-                    },
-                    timeout=min(timeout, 30),
-                )
-                if r.status_code == 200 and len(r.text) > 1000:
-                    return _parse_campaign_html(r.text, url, "wayback")
-    except Exception as e:
-        print(f"[kickstarter_fetcher] Wayback: {e}")
-    return None
-
-
-# ═══════════════════════════════════════════════════════════════
-# Stage 4: Google Cache
-# ═══════════════════════════════════════════════════════════════
-
-def _stage_google_cache(url: str, timeout: int) -> Optional[KickstarterCampaign]:
-    """Fetch via Google Web Cache."""
-    import requests
-
-    try:
-        cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}"
-        r = requests.get(
-            cache_url,
-            headers={
-                "User-Agent": (
-                    "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
-                    "Gecko/20100101 Firefox/128.0"
-                ),
-                "Accept-Language": "en-US,en;q=0.9",
-            },
-            timeout=min(timeout, 30),
-        )
-        if r.status_code == 200 and len(r.text) > 1000:
-            if "before you continue" in r.text.lower() or "cookie" in r.text.lower()[:2000]:
-                return None
-            return _parse_campaign_html(r.text, url, "google_cache")
-    except Exception as e:
-        print(f"[kickstarter_fetcher] Google Cache: {e}")
-    return None
-
-
-# ═══════════════════════════════════════════════════════════════
-# Stage 5: Press Sources (Title-Only)
-# ═══════════════════════════════════════════════════════════════
-
-def _stage_press_sources(url: str, timeout: int = 10) -> Optional[KickstarterCampaign]:
-    """Extract project name from URL slug."""
-    slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
-    if not slug_match:
         return None
-    slug = slug_match.group(1)
+
-    project_name = slug.split("/")[-1].replace("-", " ").title()
+    if data.get("status") != "ok":
+        print(f"[kickstarter_fetcher] FlareSolverr error: {data.get('message', '?')}")
+        return None
+
+    sol = data.get("solution", {})
+    html = sol.get("response", "")
+    if not html or len(html) < 5000:
+        print(f"[kickstarter_fetcher] Empty/short response: {len(html)} bytes")
+        return None
+
+    # Check for Cloudflare block in rendered page
+    if "Just a moment" in html[:2000]:
+        print("[kickstarter_fetcher] Cloudflare block after rendering")
+        return None
+
+    return _parse_campaign_native(html, url, "flaresolverr")
+
+
+def _parse_campaign_native(
+    html: str, url: str, source: str
+) -> Optional[KickstarterCampaign]:
+    """
+    Parse Kickstarter campaign data from embedded flat JSON fields.
+    Kickstarter injects data like:
+      "project_name":"Earthborne Trailblazer"
+      "project_backers_count":3969
+      "project_current_amount_pledged_usd":549130.0
+      "project_goal_usd":150000.0
+      "project_state":"successful"
+      "project_deadline":"2026-06-16T11:01:05-04:00"
+      "project_blurb":"Explore the wilderness..."
+    """
+
+    def _re_first(pattern: str, html: str, group: int = 1) -> Optional[str]:
+        m = re.search(pattern, html)
+        return m.group(group) if m else None
+
+    def _re_int(pattern: str, html: str) -> int:
+        m = re.search(pattern, html)
+        return int(m.group(1)) if m else 0
+
+    def _re_float(pattern: str, html: str) -> float:
+        m = re.search(pattern, html)
+        return float(m.group(1)) if m else 0.0
+
+    # Extract all fields
+    name = _re_first(r'"project_name":"([^"]+)"', html)
+    if not name:
+        # Try og:title as fallback
+        name = _re_first(r'<meta\s+property="og:title"\s+content="([^"]+)"', html)
+    if not name:
+        return None
+
+    blurb = _re_first(r'"project_blurb":"([^"]*)"', html)
+    currency = _re_first(r'"project_currency":"([^"]+)"', html) or "USD"
+    state = _re_first(r'"project_state":"([^"]+)"', html) or "unknown"
+
+    pledged_usd = _re_float(r'"project_current_amount_pledged_usd":([\d.]+)', html)
+    goal_usd = _re_float(r'"project_goal_usd":([\d.]+)', html)
+    backers = _re_int(r'"project_backers_count":(\d+)', html)
+    deadline = _re_first(r'"project_deadline":"([^"]+)"', html)
+
+    # Format amounts
+    def fmt_amount(usd_val: float) -> str:
+        return f"${usd_val:,.0f}"
+
+    # Calculate days left
+    days_left = -1
+    if deadline:
+        try:
+            from datetime import datetime
+            dt = datetime.fromisoformat(deadline.replace("Z", "+00:00"))
+            delta = dt - datetime.now(dt.tzinfo)
+            days_left = max(0, delta.days)
+        except Exception:
+            pass
 
     return KickstarterCampaign(
         url=url,
-        title=project_name,
+        title=name,
-        source="press",
+        pledged=fmt_amount(pledged_usd),
-        status="unknown",
+        goal=fmt_amount(goal_usd),
-        description=(
+        backers=backers,
-            f"[Indirekte Quelle — keine Live-Daten verfügbar. "
+        days_left=days_left if state == "live" else -1,
-            f"Projekt: {project_name}]"
+        status=state,
-        ),
+        description=(blurb or "")[:500],
+        source=source,
         fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
     )
 
 
 # ═══════════════════════════════════════════════════════════════
-# HTML Parser
+# Stage 1: URL-Slug Fallback
 # ═══════════════════════════════════════════════════════════════
 
-def _parse_campaign_html(
+def _stage_slug(url: str) -> KickstarterCampaign:
-    html: str, url: str, source: str
+    """Extract project name from URL slug."""
-) -> Optional[KickstarterCampaign]:
+    slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
-    """Parse Kickstarter campaign HTML to structured data."""
+    slug = slug_match.group(1) if slug_match else url.rstrip("/").split("/")[-1]
-    from bs4 import BeautifulSoup
+    project_name = slug.split("/")[-1].replace("-", " ").title()
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    # ── Title ───────────────────────────────────────────────
-    title = ""
-    if source in ("wayback", "google_cache"):
-        og = soup.find("meta", property="og:title")
-        if og and og.get("content"):
-            title = og["content"].strip()
-        if not title:
-            h1 = soup.find("h1")
-            if h1:
-                title = h1.get_text(" ", strip=True)[:200]
-        if not title:
-            for tag in soup.find_all(["h2", "h3"]):
-                txt = tag.get_text(" ", strip=True)
-                if len(txt) > 15:
-                    title = txt[:200]
-                    break
-
-    if not title:
-        title_tag = soup.find("title")
-        if title_tag:
-            title = title_tag.text.strip()
-            title = re.sub(
-                r"\s*by\s+.*?—\s*Kickstarter\s*$", "", title, flags=re.I
-            ).strip()
-            if title.lower() in ("wayback machine", "just a moment..."):
-                title = ""
-
-    if not title:
-        og = soup.find("meta", property="og:title")
-        if og and og.get("content"):
-            title = og["content"].strip()
-
-    # ── JSON-LD ─────────────────────────────────────────────
-    pledged = ""
-    goal = ""
-    backers = 0
-    description = ""
-    image_url = ""
-
-    for script in soup.find_all("script", type="application/ld+json"):
-        try:
-            data = json.loads(script.string)
-            if not isinstance(data, dict):
-                continue
-            items = data.get("@graph", [data])
-            for item in items:
-                if isinstance(item, dict):
-                    if not title and item.get("name"):
-                        title = item["name"]
-                    if not description and item.get("description"):
-                        description = str(item["description"])[:500]
-                    if item.get("image") and not image_url:
-                        img = item["image"]
-                        if isinstance(img, list):
-                            img = img[0] if img else ""
-                        if isinstance(img, str):
-                            image_url = img
-        except (json.JSONDecodeError, AttributeError, TypeError):
-            pass
-
-    # ── Text regex ──────────────────────────────────────────
-    text = soup.get_text(" ", strip=True)
-
-    for pat in [
-        r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:pledged|von|zugesagt)",
-        r"([\d,.]+)\s*(?:€|EUR|USD|\$|£)\s*pledged",
-    ]:
-        m = re.search(pat, text, re.I)
-        if m:
-            pledged = m.group(0).strip()
-            break
-
-    for pat in [
-        r"(?:goal|Ziel)\s*(?:of|von)?\s*(?:€|EUR|USD|\$|£)\s*([\d,.]+)",
-        r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:goal|Ziel)",
-    ]:
-        m = re.search(pat, text, re.I)
-        if m:
-            goal = m.group(0).strip()
-            break
-
-    backers_match = re.search(r"([\d,]+)\s*backers?", text, re.I)
-    if backers_match:
-        try:
-            backers = int(backers_match.group(1).replace(",", ""))
-        except ValueError:
-            pass
-
-    days_left = -1
-    days_match = re.search(
-        r"(\d+)\s*(?:days?|Tage)\s*(?:to go|left|verbleibend)", text, re.I
-    )
-    if days_match:
-        try:
-            days_left = int(days_match.group(1))
-        except ValueError:
-            pass
-
-    # ── Status ──────────────────────────────────────────────
-    status = "unknown"
-    text_lower = text.lower()
-    if any(t in text_lower for t in ["successfully funded", "erfolgreich finanziert"]):
-        status = "funded"
-    elif any(t in text_lower for t in ["cancelled", "abgebrochen"]):
-        status = "cancelled"
-    elif days_left >= 0:
-        status = "live"
-
-    if not description:
-        meta_desc = soup.find("meta", attrs={"name": "description"})
-        if meta_desc and meta_desc.get("content"):
-            description = meta_desc["content"][:500]
-
-    if not title:
-        return None
 
     return KickstarterCampaign(
         url=url,
-        title=title,
+        title=project_name,
-        pledged=pledged,
+        source="slug",
-        goal=goal,
+        status="unknown",
-        backers=backers,
-        days_left=days_left,
-        status=status,
-        description=description,
-        image_url=image_url,
-        source=source,
-        raw_html=text[:30000],
         fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
     )
 
@@ -448,10 +237,10 @@ if __name__ == "__main__":
 
     if len(sys.argv) < 2:
         print("Usage: python kickstarter_fetcher.py <kickstarter-url>")
-        print("\nStatus: Alle Stages blockiert. Nur Titel-Fallback (Stage 5) liefert Daten.")
+        print("\nRequires FlareSolverr running on localhost:8191")
         sys.exit(1)
 
     url = sys.argv[1]
     print(f"Fetching: {url}")
-    print("-" * 60)
+    print("-" * 50)
     print(status_report(fetch_campaign(url)))