diff --git a/kickstarter_fetcher.py b/kickstarter_fetcher.py index 0ea6e0b..2817b09 100644 --- a/kickstarter_fetcher.py +++ b/kickstarter_fetcher.py @@ -2,66 +2,52 @@ """ Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy =========================================================== -Tested 2026-06-20 — Results: +Final version 2026-06-21 — FlareSolverr breakthrough! - Stage 1: Playwright + iproyal Proxy → ❌ HTTP 403 (Proxy erkannt) - Stage 2: Tavily Extract API → ❌ "Failed to fetch url" - Stage 3: Wayback Machine → ❌ Archiviert nur 403-Seiten - Stage 4: Google Cache → ⚠️ EU-Cookie-Wall blockiert - Stage 5: Press Sources (URL-Slug) → ✅ Titel (keine Funding-Daten) - Stage X: Kicktraq API → 🔮 Zu evaluieren (Drittanbieter) +Stages: + 0. FlareSolverr (localhost:8191) → LIVE campaign data! ✅ + 1. Fallback: URL-Slug → Titel (immer) -ACTION REQUIRED: Keine der Stages liefert Live-Funding-Daten. - Option A: Bright Data / Oxylabs Web Unlocker (Kickstarter-spezifisch) - Option B: Kicktraq API (https://www.kicktraq.com/api/) - Option C: Manuell recherchieren (Autor öffnet Kampagne im Browser) +How it works: + FlareSolverr runs a headless Chromium that solves Cloudflare challenges. + Kickstarter embeds campaign data as flat JSON fields in the HTML: + project_name, project_backers_count, project_current_amount_pledged_usd, etc. + We extract these via regex → structured KickstarterCampaign object. Usage: from kickstarter_fetcher import fetch_campaign, status_report c = fetch_campaign("https://www.kickstarter.com/projects/...") print(status_report(c)) -Author: Cody (Hermes Agent), 2026-06-20 +Author: Cody (Hermes Agent), 2026-06-20/21 """ from dataclasses import dataclass from typing import Optional -import os -import re import json +import re import time + # ═══════════════════════════════════════════════════════════════ # Configuration # ═══════════════════════════════════════════════════════════════ -# iproyal Residential Proxy — ERKANNT von Kickstarter (HTTP 403) -# Nur aktivieren wenn ein besserer Proxy verfügbar ist -PROXY_CONFIG = { - "server": "http://8B7M8nJOfXTNHPUw:lR1VidJhN5RrRbeK_region-northamerica@geo.iproyal.com:12321", -} -PROXY_ENABLED = False # Auf True setzen wenn ein unblockter Proxy verfügbar ist - -TAVILY_API_KEY = os.environ.get( - "TAVILY_API_KEY", - "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", -) -TAVILY_ENABLED = True # Wieder aktivieren wenn Tavily Kickstarter unterstützt +FLARESOLVERR_URL = "http://localhost:8191/v1" +FLARESOLVERR_TIMEOUT = 60 # seconds for KS page load @dataclass class KickstarterCampaign: url: str title: str = "" - pledged: str = "" - goal: str = "" + pledged: str = "" # e.g. "$549,130" + goal: str = "" # e.g. "$150,000" backers: int = 0 days_left: int = -1 - status: str = "unknown" + status: str = "unknown" # "live", "successful", "cancelled" description: str = "" - image_url: str = "" - source: str = "" - raw_html: str = "" + source: str = "" # "flaresolverr", "slug" fetched_at: str = "" @@ -69,372 +55,175 @@ class KickstarterCampaign: # Public API # ═══════════════════════════════════════════════════════════════ -def fetch_campaign(url: str, timeout: int = 45) -> Optional[KickstarterCampaign]: +def fetch_campaign(url: str, timeout: int = 90) -> Optional[KickstarterCampaign]: """ - Fetch a Kickstarter campaign page with multi-stage fallback. - Returns KickstarterCampaign or None if all stages fail. + Fetch a Kickstarter campaign page. + Stage 0: FlareSolverr (LIVE data via headless Chromium) + Stage 1: URL-Slug fallback (title only) """ - # Stage 1: Playwright via Residential Proxy (nur wenn aktiviert) - if PROXY_ENABLED: - result = _stage_playwright(url, timeout) - if result: - return result - - # Stage 2: Tavily Extract (nur wenn aktiviert) - if TAVILY_ENABLED: - result = _stage_tavily(url, timeout) - if result: - return result - - # Stage 3: Wayback Machine (archiviert leider nur 403-Seiten) - result = _stage_wayback(url, timeout) - if result and result.title and "wayback machine" not in result.title.lower(): + # Stage 0: FlareSolverr + result = _stage_flaresolverr(url, timeout) + if result: return result - # Stage 4: Google Cache (EU-Cookie-Wall blockiert) - result = _stage_google_cache(url, timeout) - if result and "bevor sie zu google" not in result.title.lower(): - return result - - # Stage 5: Press Sources (Titel aus URL-Slug) - return _stage_press_sources(url, timeout) + # Stage 1: Slug + return _stage_slug(url) def status_report(campaign: Optional[KickstarterCampaign]) -> str: """Generate a human-readable status report.""" if not campaign: - return "❌ KEINE DATEN — Alle Stages fehlgeschlagen." + return "❌ KEINE DATEN" - if campaign.source == "press": + if campaign.source == "slug": return ( - f"⚠️ NUR TITEL (Presse-Fallback)\n" + f"⚠️ NUR TITEL\n" f" {campaign.title}\n" - f" Keine Live-Daten verfügbar.\n" f" → Bitte manuell recherchieren: {campaign.url}" ) return ( - f"✅ {campaign.source.upper()}\n" + f"✅ LIVE ({campaign.source})\n" f" {campaign.title}\n" f" {campaign.pledged} von {campaign.goal}\n" - f" {campaign.backers} Unterstützer\n" - f" {campaign.days_left} Tage verbleibend\n" + f" {campaign.backers:,} Unterstützer\n" f" Status: {campaign.status}" ) # ═══════════════════════════════════════════════════════════════ -# Stage 1: Playwright Headless Browser via Proxy +# Stage 0: FlareSolverr → Headless Chromium → KS Page → Parse # ═══════════════════════════════════════════════════════════════ -def _stage_playwright(url: str, timeout: int) -> Optional[KickstarterCampaign]: - """Fetch via headless Chromium with Residential Proxy.""" - try: - from playwright.sync_api import sync_playwright - except ImportError: - return None - - try: - with sync_playwright() as p: - browser = p.chromium.launch( - headless=True, - args=[ - "--no-sandbox", - "--disable-setuid-sandbox", - "--disable-dev-shm-usage", - "--disable-gpu", - ], - ) - ctx_kwargs = { - "user_agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/125.0.0.0 Safari/537.36" - ), - "viewport": {"width": 1920, "height": 1080}, - } - if PROXY_ENABLED: - ctx_kwargs["proxy"] = PROXY_CONFIG - - context = browser.new_context(**ctx_kwargs) - page = context.new_page() - page.goto(url, timeout=timeout * 1000, wait_until="domcontentloaded") - page.wait_for_timeout(8000) - html = page.content() - browser.close() - - if "Just a moment" in html: - return None - return _parse_campaign_html(html, url, "playwright") - - except Exception as e: - print(f"[kickstarter_fetcher] Playwright: {e}") - return None - - -# ═══════════════════════════════════════════════════════════════ -# Stage 2: Tavily Extract API -# ═══════════════════════════════════════════════════════════════ - -def _stage_tavily(url: str, timeout: int) -> Optional[KickstarterCampaign]: - """Fetch via Tavily Extract API.""" +def _stage_flaresolverr(url: str, timeout: int) -> Optional[KickstarterCampaign]: + """ + Fetch Kickstarter page via FlareSolverr (headless Chromium). + Extracts campaign data from embedded flat JSON fields. + """ import requests try: r = requests.post( - "https://api.tavily.com/extract", + FLARESOLVERR_URL, json={ - "api_key": TAVILY_API_KEY, - "urls": [url], - "extract_depth": "advanced", + "cmd": "request.get", + "url": url, + "maxTimeout": min(timeout, FLARESOLVERR_TIMEOUT) * 1000, }, - timeout=min(timeout, 30), + timeout=timeout + 10, ) data = r.json() - results = data.get("results", []) - if results and results[0].get("raw_content"): - html = results[0]["raw_content"] - if "Just a moment" in html: - return None - return _parse_campaign_html(html, url, "tavily") except Exception as e: - print(f"[kickstarter_fetcher] Tavily: {e}") - return None - - -# ═══════════════════════════════════════════════════════════════ -# Stage 3: Wayback Machine -# ═══════════════════════════════════════════════════════════════ - -def _stage_wayback(url: str, timeout: int) -> Optional[KickstarterCampaign]: - """Fetch most recent Wayback Machine snapshot.""" - import requests - - try: - cdx_url = ( - f"https://web.archive.org/web/timemap/json?" - f"url={url}&matchType=prefix&limit=5" - ) - r = requests.get(cdx_url, timeout=10) - snapshots = r.json() - # Filter 403 snapshots, find first non-403 - for snap in snapshots[1:]: # skip header - if len(snap) >= 5 and str(snap[4]) != "403": - ts = snap[1] - snapshot_url = f"https://web.archive.org/web/{ts}/{url}" - r = requests.get( - snapshot_url, - headers={ - "User-Agent": ( - "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) " - "Gecko/20100101 Firefox/128.0" - ), - }, - timeout=min(timeout, 30), - ) - if r.status_code == 200 and len(r.text) > 1000: - return _parse_campaign_html(r.text, url, "wayback") - except Exception as e: - print(f"[kickstarter_fetcher] Wayback: {e}") - return None - - -# ═══════════════════════════════════════════════════════════════ -# Stage 4: Google Cache -# ═══════════════════════════════════════════════════════════════ - -def _stage_google_cache(url: str, timeout: int) -> Optional[KickstarterCampaign]: - """Fetch via Google Web Cache.""" - import requests - - try: - cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}" - r = requests.get( - cache_url, - headers={ - "User-Agent": ( - "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) " - "Gecko/20100101 Firefox/128.0" - ), - "Accept-Language": "en-US,en;q=0.9", - }, - timeout=min(timeout, 30), - ) - if r.status_code == 200 and len(r.text) > 1000: - if "before you continue" in r.text.lower() or "cookie" in r.text.lower()[:2000]: - return None - return _parse_campaign_html(r.text, url, "google_cache") - except Exception as e: - print(f"[kickstarter_fetcher] Google Cache: {e}") - return None - - -# ═══════════════════════════════════════════════════════════════ -# Stage 5: Press Sources (Title-Only) -# ═══════════════════════════════════════════════════════════════ - -def _stage_press_sources(url: str, timeout: int = 10) -> Optional[KickstarterCampaign]: - """Extract project name from URL slug.""" - slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url) - if not slug_match: + print(f"[kickstarter_fetcher] FlareSolverr request error: {e}") return None - slug = slug_match.group(1) - project_name = slug.split("/")[-1].replace("-", " ").title() + + if data.get("status") != "ok": + print(f"[kickstarter_fetcher] FlareSolverr error: {data.get('message', '?')}") + return None + + sol = data.get("solution", {}) + html = sol.get("response", "") + if not html or len(html) < 5000: + print(f"[kickstarter_fetcher] Empty/short response: {len(html)} bytes") + return None + + # Check for Cloudflare block in rendered page + if "Just a moment" in html[:2000]: + print("[kickstarter_fetcher] Cloudflare block after rendering") + return None + + return _parse_campaign_native(html, url, "flaresolverr") + + +def _parse_campaign_native( + html: str, url: str, source: str +) -> Optional[KickstarterCampaign]: + """ + Parse Kickstarter campaign data from embedded flat JSON fields. + Kickstarter injects data like: + "project_name":"Earthborne Trailblazer" + "project_backers_count":3969 + "project_current_amount_pledged_usd":549130.0 + "project_goal_usd":150000.0 + "project_state":"successful" + "project_deadline":"2026-06-16T11:01:05-04:00" + "project_blurb":"Explore the wilderness..." + """ + + def _re_first(pattern: str, html: str, group: int = 1) -> Optional[str]: + m = re.search(pattern, html) + return m.group(group) if m else None + + def _re_int(pattern: str, html: str) -> int: + m = re.search(pattern, html) + return int(m.group(1)) if m else 0 + + def _re_float(pattern: str, html: str) -> float: + m = re.search(pattern, html) + return float(m.group(1)) if m else 0.0 + + # Extract all fields + name = _re_first(r'"project_name":"([^"]+)"', html) + if not name: + # Try og:title as fallback + name = _re_first(r' str: + return f"${usd_val:,.0f}" + + # Calculate days left + days_left = -1 + if deadline: + try: + from datetime import datetime + dt = datetime.fromisoformat(deadline.replace("Z", "+00:00")) + delta = dt - datetime.now(dt.tzinfo) + days_left = max(0, delta.days) + except Exception: + pass return KickstarterCampaign( url=url, - title=project_name, - source="press", - status="unknown", - description=( - f"[Indirekte Quelle — keine Live-Daten verfügbar. " - f"Projekt: {project_name}]" - ), + title=name, + pledged=fmt_amount(pledged_usd), + goal=fmt_amount(goal_usd), + backers=backers, + days_left=days_left if state == "live" else -1, + status=state, + description=(blurb or "")[:500], + source=source, fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"), ) # ═══════════════════════════════════════════════════════════════ -# HTML Parser +# Stage 1: URL-Slug Fallback # ═══════════════════════════════════════════════════════════════ -def _parse_campaign_html( - html: str, url: str, source: str -) -> Optional[KickstarterCampaign]: - """Parse Kickstarter campaign HTML to structured data.""" - from bs4 import BeautifulSoup - - soup = BeautifulSoup(html, "html.parser") - - # ── Title ─────────────────────────────────────────────── - title = "" - if source in ("wayback", "google_cache"): - og = soup.find("meta", property="og:title") - if og and og.get("content"): - title = og["content"].strip() - if not title: - h1 = soup.find("h1") - if h1: - title = h1.get_text(" ", strip=True)[:200] - if not title: - for tag in soup.find_all(["h2", "h3"]): - txt = tag.get_text(" ", strip=True) - if len(txt) > 15: - title = txt[:200] - break - - if not title: - title_tag = soup.find("title") - if title_tag: - title = title_tag.text.strip() - title = re.sub( - r"\s*by\s+.*?—\s*Kickstarter\s*$", "", title, flags=re.I - ).strip() - if title.lower() in ("wayback machine", "just a moment..."): - title = "" - - if not title: - og = soup.find("meta", property="og:title") - if og and og.get("content"): - title = og["content"].strip() - - # ── JSON-LD ───────────────────────────────────────────── - pledged = "" - goal = "" - backers = 0 - description = "" - image_url = "" - - for script in soup.find_all("script", type="application/ld+json"): - try: - data = json.loads(script.string) - if not isinstance(data, dict): - continue - items = data.get("@graph", [data]) - for item in items: - if isinstance(item, dict): - if not title and item.get("name"): - title = item["name"] - if not description and item.get("description"): - description = str(item["description"])[:500] - if item.get("image") and not image_url: - img = item["image"] - if isinstance(img, list): - img = img[0] if img else "" - if isinstance(img, str): - image_url = img - except (json.JSONDecodeError, AttributeError, TypeError): - pass - - # ── Text regex ────────────────────────────────────────── - text = soup.get_text(" ", strip=True) - - for pat in [ - r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:pledged|von|zugesagt)", - r"([\d,.]+)\s*(?:€|EUR|USD|\$|£)\s*pledged", - ]: - m = re.search(pat, text, re.I) - if m: - pledged = m.group(0).strip() - break - - for pat in [ - r"(?:goal|Ziel)\s*(?:of|von)?\s*(?:€|EUR|USD|\$|£)\s*([\d,.]+)", - r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:goal|Ziel)", - ]: - m = re.search(pat, text, re.I) - if m: - goal = m.group(0).strip() - break - - backers_match = re.search(r"([\d,]+)\s*backers?", text, re.I) - if backers_match: - try: - backers = int(backers_match.group(1).replace(",", "")) - except ValueError: - pass - - days_left = -1 - days_match = re.search( - r"(\d+)\s*(?:days?|Tage)\s*(?:to go|left|verbleibend)", text, re.I - ) - if days_match: - try: - days_left = int(days_match.group(1)) - except ValueError: - pass - - # ── Status ────────────────────────────────────────────── - status = "unknown" - text_lower = text.lower() - if any(t in text_lower for t in ["successfully funded", "erfolgreich finanziert"]): - status = "funded" - elif any(t in text_lower for t in ["cancelled", "abgebrochen"]): - status = "cancelled" - elif days_left >= 0: - status = "live" - - if not description: - meta_desc = soup.find("meta", attrs={"name": "description"}) - if meta_desc and meta_desc.get("content"): - description = meta_desc["content"][:500] - - if not title: - return None +def _stage_slug(url: str) -> KickstarterCampaign: + """Extract project name from URL slug.""" + slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url) + slug = slug_match.group(1) if slug_match else url.rstrip("/").split("/")[-1] + project_name = slug.split("/")[-1].replace("-", " ").title() return KickstarterCampaign( url=url, - title=title, - pledged=pledged, - goal=goal, - backers=backers, - days_left=days_left, - status=status, - description=description, - image_url=image_url, - source=source, - raw_html=text[:30000], + title=project_name, + source="slug", + status="unknown", fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"), ) @@ -448,10 +237,10 @@ if __name__ == "__main__": if len(sys.argv) < 2: print("Usage: python kickstarter_fetcher.py ") - print("\nStatus: Alle Stages blockiert. Nur Titel-Fallback (Stage 5) liefert Daten.") + print("\nRequires FlareSolverr running on localhost:8191") sys.exit(1) url = sys.argv[1] print(f"Fetching: {url}") - print("-" * 60) + print("-" * 50) print(status_report(fetch_campaign(url)))