diff --git a/kickstarter_fetcher.py b/kickstarter_fetcher.py new file mode 100644 index 0000000..0ea6e0b --- /dev/null +++ b/kickstarter_fetcher.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +""" +Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy +=========================================================== +Tested 2026-06-20 — Results: + + Stage 1: Playwright + iproyal Proxy → ❌ HTTP 403 (Proxy erkannt) + Stage 2: Tavily Extract API → ❌ "Failed to fetch url" + Stage 3: Wayback Machine → ❌ Archiviert nur 403-Seiten + Stage 4: Google Cache → ⚠️ EU-Cookie-Wall blockiert + Stage 5: Press Sources (URL-Slug) → ✅ Titel (keine Funding-Daten) + Stage X: Kicktraq API → 🔮 Zu evaluieren (Drittanbieter) + +ACTION REQUIRED: Keine der Stages liefert Live-Funding-Daten. + Option A: Bright Data / Oxylabs Web Unlocker (Kickstarter-spezifisch) + Option B: Kicktraq API (https://www.kicktraq.com/api/) + Option C: Manuell recherchieren (Autor öffnet Kampagne im Browser) + +Usage: + from kickstarter_fetcher import fetch_campaign, status_report + c = fetch_campaign("https://www.kickstarter.com/projects/...") + print(status_report(c)) + +Author: Cody (Hermes Agent), 2026-06-20 +""" + +from dataclasses import dataclass +from typing import Optional +import os +import re +import json +import time + +# ═══════════════════════════════════════════════════════════════ +# Configuration +# ═══════════════════════════════════════════════════════════════ + +# iproyal Residential Proxy — ERKANNT von Kickstarter (HTTP 403) +# Nur aktivieren wenn ein besserer Proxy verfügbar ist +PROXY_CONFIG = { + "server": "http://8B7M8nJOfXTNHPUw:lR1VidJhN5RrRbeK_region-northamerica@geo.iproyal.com:12321", +} +PROXY_ENABLED = False # Auf True setzen wenn ein unblockter Proxy verfügbar ist + +TAVILY_API_KEY = os.environ.get( + "TAVILY_API_KEY", + "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", +) +TAVILY_ENABLED = True # Wieder aktivieren wenn Tavily Kickstarter unterstützt + + +@dataclass +class KickstarterCampaign: + url: str + title: str = "" + pledged: str = "" + goal: str = "" + backers: int = 0 + days_left: int = -1 + status: str = "unknown" + description: str = "" + image_url: str = "" + source: str = "" + raw_html: str = "" + fetched_at: str = "" + + +# ═══════════════════════════════════════════════════════════════ +# Public API +# ═══════════════════════════════════════════════════════════════ + +def fetch_campaign(url: str, timeout: int = 45) -> Optional[KickstarterCampaign]: + """ + Fetch a Kickstarter campaign page with multi-stage fallback. + Returns KickstarterCampaign or None if all stages fail. + """ + + # Stage 1: Playwright via Residential Proxy (nur wenn aktiviert) + if PROXY_ENABLED: + result = _stage_playwright(url, timeout) + if result: + return result + + # Stage 2: Tavily Extract (nur wenn aktiviert) + if TAVILY_ENABLED: + result = _stage_tavily(url, timeout) + if result: + return result + + # Stage 3: Wayback Machine (archiviert leider nur 403-Seiten) + result = _stage_wayback(url, timeout) + if result and result.title and "wayback machine" not in result.title.lower(): + return result + + # Stage 4: Google Cache (EU-Cookie-Wall blockiert) + result = _stage_google_cache(url, timeout) + if result and "bevor sie zu google" not in result.title.lower(): + return result + + # Stage 5: Press Sources (Titel aus URL-Slug) + return _stage_press_sources(url, timeout) + + +def status_report(campaign: Optional[KickstarterCampaign]) -> str: + """Generate a human-readable status report.""" + if not campaign: + return "❌ KEINE DATEN — Alle Stages fehlgeschlagen." + + if campaign.source == "press": + return ( + f"⚠️ NUR TITEL (Presse-Fallback)\n" + f" {campaign.title}\n" + f" Keine Live-Daten verfügbar.\n" + f" → Bitte manuell recherchieren: {campaign.url}" + ) + + return ( + f"✅ {campaign.source.upper()}\n" + f" {campaign.title}\n" + f" {campaign.pledged} von {campaign.goal}\n" + f" {campaign.backers} Unterstützer\n" + f" {campaign.days_left} Tage verbleibend\n" + f" Status: {campaign.status}" + ) + + +# ═══════════════════════════════════════════════════════════════ +# Stage 1: Playwright Headless Browser via Proxy +# ═══════════════════════════════════════════════════════════════ + +def _stage_playwright(url: str, timeout: int) -> Optional[KickstarterCampaign]: + """Fetch via headless Chromium with Residential Proxy.""" + try: + from playwright.sync_api import sync_playwright + except ImportError: + return None + + try: + with sync_playwright() as p: + browser = p.chromium.launch( + headless=True, + args=[ + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + ], + ) + ctx_kwargs = { + "user_agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/125.0.0.0 Safari/537.36" + ), + "viewport": {"width": 1920, "height": 1080}, + } + if PROXY_ENABLED: + ctx_kwargs["proxy"] = PROXY_CONFIG + + context = browser.new_context(**ctx_kwargs) + page = context.new_page() + page.goto(url, timeout=timeout * 1000, wait_until="domcontentloaded") + page.wait_for_timeout(8000) + html = page.content() + browser.close() + + if "Just a moment" in html: + return None + return _parse_campaign_html(html, url, "playwright") + + except Exception as e: + print(f"[kickstarter_fetcher] Playwright: {e}") + return None + + +# ═══════════════════════════════════════════════════════════════ +# Stage 2: Tavily Extract API +# ═══════════════════════════════════════════════════════════════ + +def _stage_tavily(url: str, timeout: int) -> Optional[KickstarterCampaign]: + """Fetch via Tavily Extract API.""" + import requests + + try: + r = requests.post( + "https://api.tavily.com/extract", + json={ + "api_key": TAVILY_API_KEY, + "urls": [url], + "extract_depth": "advanced", + }, + timeout=min(timeout, 30), + ) + data = r.json() + results = data.get("results", []) + if results and results[0].get("raw_content"): + html = results[0]["raw_content"] + if "Just a moment" in html: + return None + return _parse_campaign_html(html, url, "tavily") + except Exception as e: + print(f"[kickstarter_fetcher] Tavily: {e}") + return None + + +# ═══════════════════════════════════════════════════════════════ +# Stage 3: Wayback Machine +# ═══════════════════════════════════════════════════════════════ + +def _stage_wayback(url: str, timeout: int) -> Optional[KickstarterCampaign]: + """Fetch most recent Wayback Machine snapshot.""" + import requests + + try: + cdx_url = ( + f"https://web.archive.org/web/timemap/json?" + f"url={url}&matchType=prefix&limit=5" + ) + r = requests.get(cdx_url, timeout=10) + snapshots = r.json() + # Filter 403 snapshots, find first non-403 + for snap in snapshots[1:]: # skip header + if len(snap) >= 5 and str(snap[4]) != "403": + ts = snap[1] + snapshot_url = f"https://web.archive.org/web/{ts}/{url}" + r = requests.get( + snapshot_url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ), + }, + timeout=min(timeout, 30), + ) + if r.status_code == 200 and len(r.text) > 1000: + return _parse_campaign_html(r.text, url, "wayback") + except Exception as e: + print(f"[kickstarter_fetcher] Wayback: {e}") + return None + + +# ═══════════════════════════════════════════════════════════════ +# Stage 4: Google Cache +# ═══════════════════════════════════════════════════════════════ + +def _stage_google_cache(url: str, timeout: int) -> Optional[KickstarterCampaign]: + """Fetch via Google Web Cache.""" + import requests + + try: + cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}" + r = requests.get( + cache_url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ), + "Accept-Language": "en-US,en;q=0.9", + }, + timeout=min(timeout, 30), + ) + if r.status_code == 200 and len(r.text) > 1000: + if "before you continue" in r.text.lower() or "cookie" in r.text.lower()[:2000]: + return None + return _parse_campaign_html(r.text, url, "google_cache") + except Exception as e: + print(f"[kickstarter_fetcher] Google Cache: {e}") + return None + + +# ═══════════════════════════════════════════════════════════════ +# Stage 5: Press Sources (Title-Only) +# ═══════════════════════════════════════════════════════════════ + +def _stage_press_sources(url: str, timeout: int = 10) -> Optional[KickstarterCampaign]: + """Extract project name from URL slug.""" + slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url) + if not slug_match: + return None + slug = slug_match.group(1) + project_name = slug.split("/")[-1].replace("-", " ").title() + + return KickstarterCampaign( + url=url, + title=project_name, + source="press", + status="unknown", + description=( + f"[Indirekte Quelle — keine Live-Daten verfügbar. " + f"Projekt: {project_name}]" + ), + fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"), + ) + + +# ═══════════════════════════════════════════════════════════════ +# HTML Parser +# ═══════════════════════════════════════════════════════════════ + +def _parse_campaign_html( + html: str, url: str, source: str +) -> Optional[KickstarterCampaign]: + """Parse Kickstarter campaign HTML to structured data.""" + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html, "html.parser") + + # ── Title ─────────────────────────────────────────────── + title = "" + if source in ("wayback", "google_cache"): + og = soup.find("meta", property="og:title") + if og and og.get("content"): + title = og["content"].strip() + if not title: + h1 = soup.find("h1") + if h1: + title = h1.get_text(" ", strip=True)[:200] + if not title: + for tag in soup.find_all(["h2", "h3"]): + txt = tag.get_text(" ", strip=True) + if len(txt) > 15: + title = txt[:200] + break + + if not title: + title_tag = soup.find("title") + if title_tag: + title = title_tag.text.strip() + title = re.sub( + r"\s*by\s+.*?—\s*Kickstarter\s*$", "", title, flags=re.I + ).strip() + if title.lower() in ("wayback machine", "just a moment..."): + title = "" + + if not title: + og = soup.find("meta", property="og:title") + if og and og.get("content"): + title = og["content"].strip() + + # ── JSON-LD ───────────────────────────────────────────── + pledged = "" + goal = "" + backers = 0 + description = "" + image_url = "" + + for script in soup.find_all("script", type="application/ld+json"): + try: + data = json.loads(script.string) + if not isinstance(data, dict): + continue + items = data.get("@graph", [data]) + for item in items: + if isinstance(item, dict): + if not title and item.get("name"): + title = item["name"] + if not description and item.get("description"): + description = str(item["description"])[:500] + if item.get("image") and not image_url: + img = item["image"] + if isinstance(img, list): + img = img[0] if img else "" + if isinstance(img, str): + image_url = img + except (json.JSONDecodeError, AttributeError, TypeError): + pass + + # ── Text regex ────────────────────────────────────────── + text = soup.get_text(" ", strip=True) + + for pat in [ + r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:pledged|von|zugesagt)", + r"([\d,.]+)\s*(?:€|EUR|USD|\$|£)\s*pledged", + ]: + m = re.search(pat, text, re.I) + if m: + pledged = m.group(0).strip() + break + + for pat in [ + r"(?:goal|Ziel)\s*(?:of|von)?\s*(?:€|EUR|USD|\$|£)\s*([\d,.]+)", + r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:goal|Ziel)", + ]: + m = re.search(pat, text, re.I) + if m: + goal = m.group(0).strip() + break + + backers_match = re.search(r"([\d,]+)\s*backers?", text, re.I) + if backers_match: + try: + backers = int(backers_match.group(1).replace(",", "")) + except ValueError: + pass + + days_left = -1 + days_match = re.search( + r"(\d+)\s*(?:days?|Tage)\s*(?:to go|left|verbleibend)", text, re.I + ) + if days_match: + try: + days_left = int(days_match.group(1)) + except ValueError: + pass + + # ── Status ────────────────────────────────────────────── + status = "unknown" + text_lower = text.lower() + if any(t in text_lower for t in ["successfully funded", "erfolgreich finanziert"]): + status = "funded" + elif any(t in text_lower for t in ["cancelled", "abgebrochen"]): + status = "cancelled" + elif days_left >= 0: + status = "live" + + if not description: + meta_desc = soup.find("meta", attrs={"name": "description"}) + if meta_desc and meta_desc.get("content"): + description = meta_desc["content"][:500] + + if not title: + return None + + return KickstarterCampaign( + url=url, + title=title, + pledged=pledged, + goal=goal, + backers=backers, + days_left=days_left, + status=status, + description=description, + image_url=image_url, + source=source, + raw_html=text[:30000], + fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"), + ) + + +# ═══════════════════════════════════════════════════════════════ +# CLI +# ═══════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("Usage: python kickstarter_fetcher.py ") + print("\nStatus: Alle Stages blockiert. Nur Titel-Fallback (Stage 5) liefert Daten.") + sys.exit(1) + + url = sys.argv[1] + print(f"Fetching: {url}") + print("-" * 60) + print(status_report(fetch_campaign(url)))