BSN-Chatsystem/kickstarter_fetcher.py

#!/usr/bin/env python3
"""
Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy
===========================================================
Final version 2026-06-21 — FlareSolverr breakthrough!

Stages:
  0. FlareSolverr (localhost:8191) → LIVE campaign data! ✅
  1. Fallback: URL-Slug → Titel (immer)

How it works:
  FlareSolverr runs a headless Chromium that solves Cloudflare challenges.
  Kickstarter embeds campaign data as flat JSON fields in the HTML:
    project_name, project_backers_count, project_current_amount_pledged_usd, etc.
  We extract these via regex → structured KickstarterCampaign object.

Usage:
    from kickstarter_fetcher import fetch_campaign, status_report
    c = fetch_campaign("https://www.kickstarter.com/projects/...")
    print(status_report(c))

Author: Cody (Hermes Agent), 2026-06-20/21
"""

from dataclasses import dataclass
from typing import Optional
import json
import re
import time


# ═══════════════════════════════════════════════════════════════
# Configuration
# ═══════════════════════════════════════════════════════════════

FLARESOLVERR_URL = "http://localhost:8191/v1"
FLARESOLVERR_TIMEOUT = 60  # seconds for KS page load


@dataclass
class KickstarterCampaign:
    url: str
    title: str = ""
    pledged: str = ""         # e.g. "$549,130"
    goal: str = ""            # e.g. "$150,000"
    backers: int = 0
    days_left: int = -1
    status: str = "unknown"   # "live", "successful", "cancelled"
    description: str = ""
    source: str = ""          # "flaresolverr", "slug"
    fetched_at: str = ""


# ═══════════════════════════════════════════════════════════════
# Public API
# ═══════════════════════════════════════════════════════════════

def fetch_campaign(url: str, timeout: int = 90) -> Optional[KickstarterCampaign]:
    """
    Fetch a Kickstarter campaign page.
    Stage 0: FlareSolverr (LIVE data via headless Chromium)
    Stage 1: URL-Slug fallback (title only)
    """

    # Stage 0: FlareSolverr
    result = _stage_flaresolverr(url, timeout)
    if result:
        return result

    # Stage 1: Slug
    return _stage_slug(url)


def status_report(campaign: Optional[KickstarterCampaign]) -> str:
    """Generate a human-readable status report."""
    if not campaign:
        return "❌ KEINE DATEN"

    if campaign.source == "slug":
        return (
            f"⚠️  NUR TITEL\n"
            f"   {campaign.title}\n"
            f"   → Bitte manuell recherchieren: {campaign.url}"
        )

    return (
        f"✅ LIVE ({campaign.source})\n"
        f"   {campaign.title}\n"
        f"   {campaign.pledged} von {campaign.goal}\n"
        f"   {campaign.backers:,} Unterstützer\n"
        f"   Status: {campaign.status}"
    )


# ═══════════════════════════════════════════════════════════════
# Stage 0: FlareSolverr → Headless Chromium → KS Page → Parse
# ═══════════════════════════════════════════════════════════════

def _stage_flaresolverr(url: str, timeout: int) -> Optional[KickstarterCampaign]:
    """
    Fetch Kickstarter page via FlareSolverr (headless Chromium).
    Extracts campaign data from embedded flat JSON fields.
    """
    import requests

    try:
        r = requests.post(
            FLARESOLVERR_URL,
            json={
                "cmd": "request.get",
                "url": url,
                "maxTimeout": min(timeout, FLARESOLVERR_TIMEOUT) * 1000,
            },
            timeout=timeout + 10,
        )
        data = r.json()
    except Exception as e:
        print(f"[kickstarter_fetcher] FlareSolverr request error: {e}")
        return None

    if data.get("status") != "ok":
        print(f"[kickstarter_fetcher] FlareSolverr error: {data.get('message', '?')}")
        return None

    sol = data.get("solution", {})
    html = sol.get("response", "")
    if not html or len(html) < 5000:
        print(f"[kickstarter_fetcher] Empty/short response: {len(html)} bytes")
        return None

    # Check for Cloudflare block in rendered page
    if "Just a moment" in html[:2000]:
        print("[kickstarter_fetcher] Cloudflare block after rendering")
        return None

    return _parse_campaign_native(html, url, "flaresolverr")


def _parse_campaign_native(
    html: str, url: str, source: str
) -> Optional[KickstarterCampaign]:
    """
    Parse Kickstarter campaign data from embedded flat JSON fields.
    Kickstarter injects data like:
      "project_name":"Earthborne Trailblazer"
      "project_backers_count":3969
      "project_current_amount_pledged_usd":549130.0
      "project_goal_usd":150000.0
      "project_state":"successful"
      "project_deadline":"2026-06-16T11:01:05-04:00"
      "project_blurb":"Explore the wilderness..."
    """

    def _re_first(pattern: str, html: str, group: int = 1) -> Optional[str]:
        m = re.search(pattern, html)
        return m.group(group) if m else None

    def _re_int(pattern: str, html: str) -> int:
        m = re.search(pattern, html)
        return int(m.group(1)) if m else 0

    def _re_float(pattern: str, html: str) -> float:
        m = re.search(pattern, html)
        return float(m.group(1)) if m else 0.0

    # Extract all fields
    name = _re_first(r'"project_name":"([^"]+)"', html)
    if not name:
        # Try og:title as fallback
        name = _re_first(r'<meta\s+property="og:title"\s+content="([^"]+)"', html)
    if not name:
        return None

    blurb = _re_first(r'"project_blurb":"([^"]*)"', html)
    currency = _re_first(r'"project_currency":"([^"]+)"', html) or "USD"
    state = _re_first(r'"project_state":"([^"]+)"', html) or "unknown"

    pledged_usd = _re_float(r'"project_current_amount_pledged_usd":([\d.]+)', html)
    goal_usd = _re_float(r'"project_goal_usd":([\d.]+)', html)
    backers = _re_int(r'"project_backers_count":(\d+)', html)
    deadline = _re_first(r'"project_deadline":"([^"]+)"', html)

    # Format amounts
    def fmt_amount(usd_val: float) -> str:
        return f"${usd_val:,.0f}"

    # Calculate days left
    days_left = -1
    if deadline:
        try:
            from datetime import datetime
            dt = datetime.fromisoformat(deadline.replace("Z", "+00:00"))
            delta = dt - datetime.now(dt.tzinfo)
            days_left = max(0, delta.days)
        except Exception:
            pass

    return KickstarterCampaign(
        url=url,
        title=name,
        pledged=fmt_amount(pledged_usd),
        goal=fmt_amount(goal_usd),
        backers=backers,
        days_left=days_left if state == "live" else -1,
        status=state,
        description=(blurb or "")[:500],
        source=source,
        fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
    )


# ═══════════════════════════════════════════════════════════════
# Stage 1: URL-Slug Fallback
# ═══════════════════════════════════════════════════════════════

def _stage_slug(url: str) -> KickstarterCampaign:
    """Extract project name from URL slug."""
    slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
    slug = slug_match.group(1) if slug_match else url.rstrip("/").split("/")[-1]
    project_name = slug.split("/")[-1].replace("-", " ").title()

    return KickstarterCampaign(
        url=url,
        title=project_name,
        source="slug",
        status="unknown",
        fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
    )


# ═══════════════════════════════════════════════════════════════
# CLI
# ═══════════════════════════════════════════════════════════════

if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2:
        print("Usage: python kickstarter_fetcher.py <kickstarter-url>")
        print("\nRequires FlareSolverr running on localhost:8191")
        sys.exit(1)

    url = sys.argv[1]
    print(f"Fetching: {url}")
    print("-" * 50)
    print(status_report(fetch_campaign(url)))