Hermes Agent
59b67fa79a
Tests 2026-06-20: Stage 1: Playwright+iproyal → ❌ HTTP 403 (Proxy von KS erkannt) Stage 2: Tavily Extract → ❌ Failed to fetch Stage 3: Wayback Machine → ✅ Titel (Filter für non-403 Snapshots) Stage 4: Google Cache → ⚠️ EU-Cookie-Wall Stage 5: Press/Titel → ✅ URL-Slug PROXY_ENABLED=False, TAVILY_ENABLED=True (Flags für spätere Aktivierung) Option A: Bright Data/Oxylabs Web Unlocker Option B: Kicktraq API Option C: Manuelle Recherche
458 lines
18 KiB
Python
458 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy
|
|
===========================================================
|
|
Tested 2026-06-20 — Results:
|
|
|
|
Stage 1: Playwright + iproyal Proxy → ❌ HTTP 403 (Proxy erkannt)
|
|
Stage 2: Tavily Extract API → ❌ "Failed to fetch url"
|
|
Stage 3: Wayback Machine → ❌ Archiviert nur 403-Seiten
|
|
Stage 4: Google Cache → ⚠️ EU-Cookie-Wall blockiert
|
|
Stage 5: Press Sources (URL-Slug) → ✅ Titel (keine Funding-Daten)
|
|
Stage X: Kicktraq API → 🔮 Zu evaluieren (Drittanbieter)
|
|
|
|
ACTION REQUIRED: Keine der Stages liefert Live-Funding-Daten.
|
|
Option A: Bright Data / Oxylabs Web Unlocker (Kickstarter-spezifisch)
|
|
Option B: Kicktraq API (https://www.kicktraq.com/api/)
|
|
Option C: Manuell recherchieren (Autor öffnet Kampagne im Browser)
|
|
|
|
Usage:
|
|
from kickstarter_fetcher import fetch_campaign, status_report
|
|
c = fetch_campaign("https://www.kickstarter.com/projects/...")
|
|
print(status_report(c))
|
|
|
|
Author: Cody (Hermes Agent), 2026-06-20
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Optional
|
|
import os
|
|
import re
|
|
import json
|
|
import time
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Configuration
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
# iproyal Residential Proxy — ERKANNT von Kickstarter (HTTP 403)
|
|
# Nur aktivieren wenn ein besserer Proxy verfügbar ist
|
|
PROXY_CONFIG = {
|
|
"server": "http://8B7M8nJOfXTNHPUw:lR1VidJhN5RrRbeK_region-northamerica@geo.iproyal.com:12321",
|
|
}
|
|
PROXY_ENABLED = False # Auf True setzen wenn ein unblockter Proxy verfügbar ist
|
|
|
|
TAVILY_API_KEY = os.environ.get(
|
|
"TAVILY_API_KEY",
|
|
"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",
|
|
)
|
|
TAVILY_ENABLED = True # Wieder aktivieren wenn Tavily Kickstarter unterstützt
|
|
|
|
|
|
@dataclass
|
|
class KickstarterCampaign:
|
|
url: str
|
|
title: str = ""
|
|
pledged: str = ""
|
|
goal: str = ""
|
|
backers: int = 0
|
|
days_left: int = -1
|
|
status: str = "unknown"
|
|
description: str = ""
|
|
image_url: str = ""
|
|
source: str = ""
|
|
raw_html: str = ""
|
|
fetched_at: str = ""
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Public API
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def fetch_campaign(url: str, timeout: int = 45) -> Optional[KickstarterCampaign]:
|
|
"""
|
|
Fetch a Kickstarter campaign page with multi-stage fallback.
|
|
Returns KickstarterCampaign or None if all stages fail.
|
|
"""
|
|
|
|
# Stage 1: Playwright via Residential Proxy (nur wenn aktiviert)
|
|
if PROXY_ENABLED:
|
|
result = _stage_playwright(url, timeout)
|
|
if result:
|
|
return result
|
|
|
|
# Stage 2: Tavily Extract (nur wenn aktiviert)
|
|
if TAVILY_ENABLED:
|
|
result = _stage_tavily(url, timeout)
|
|
if result:
|
|
return result
|
|
|
|
# Stage 3: Wayback Machine (archiviert leider nur 403-Seiten)
|
|
result = _stage_wayback(url, timeout)
|
|
if result and result.title and "wayback machine" not in result.title.lower():
|
|
return result
|
|
|
|
# Stage 4: Google Cache (EU-Cookie-Wall blockiert)
|
|
result = _stage_google_cache(url, timeout)
|
|
if result and "bevor sie zu google" not in result.title.lower():
|
|
return result
|
|
|
|
# Stage 5: Press Sources (Titel aus URL-Slug)
|
|
return _stage_press_sources(url, timeout)
|
|
|
|
|
|
def status_report(campaign: Optional[KickstarterCampaign]) -> str:
|
|
"""Generate a human-readable status report."""
|
|
if not campaign:
|
|
return "❌ KEINE DATEN — Alle Stages fehlgeschlagen."
|
|
|
|
if campaign.source == "press":
|
|
return (
|
|
f"⚠️ NUR TITEL (Presse-Fallback)\n"
|
|
f" {campaign.title}\n"
|
|
f" Keine Live-Daten verfügbar.\n"
|
|
f" → Bitte manuell recherchieren: {campaign.url}"
|
|
)
|
|
|
|
return (
|
|
f"✅ {campaign.source.upper()}\n"
|
|
f" {campaign.title}\n"
|
|
f" {campaign.pledged} von {campaign.goal}\n"
|
|
f" {campaign.backers} Unterstützer\n"
|
|
f" {campaign.days_left} Tage verbleibend\n"
|
|
f" Status: {campaign.status}"
|
|
)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Stage 1: Playwright Headless Browser via Proxy
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def _stage_playwright(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
|
"""Fetch via headless Chromium with Residential Proxy."""
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
except ImportError:
|
|
return None
|
|
|
|
try:
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(
|
|
headless=True,
|
|
args=[
|
|
"--no-sandbox",
|
|
"--disable-setuid-sandbox",
|
|
"--disable-dev-shm-usage",
|
|
"--disable-gpu",
|
|
],
|
|
)
|
|
ctx_kwargs = {
|
|
"user_agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/125.0.0.0 Safari/537.36"
|
|
),
|
|
"viewport": {"width": 1920, "height": 1080},
|
|
}
|
|
if PROXY_ENABLED:
|
|
ctx_kwargs["proxy"] = PROXY_CONFIG
|
|
|
|
context = browser.new_context(**ctx_kwargs)
|
|
page = context.new_page()
|
|
page.goto(url, timeout=timeout * 1000, wait_until="domcontentloaded")
|
|
page.wait_for_timeout(8000)
|
|
html = page.content()
|
|
browser.close()
|
|
|
|
if "Just a moment" in html:
|
|
return None
|
|
return _parse_campaign_html(html, url, "playwright")
|
|
|
|
except Exception as e:
|
|
print(f"[kickstarter_fetcher] Playwright: {e}")
|
|
return None
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Stage 2: Tavily Extract API
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def _stage_tavily(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
|
"""Fetch via Tavily Extract API."""
|
|
import requests
|
|
|
|
try:
|
|
r = requests.post(
|
|
"https://api.tavily.com/extract",
|
|
json={
|
|
"api_key": TAVILY_API_KEY,
|
|
"urls": [url],
|
|
"extract_depth": "advanced",
|
|
},
|
|
timeout=min(timeout, 30),
|
|
)
|
|
data = r.json()
|
|
results = data.get("results", [])
|
|
if results and results[0].get("raw_content"):
|
|
html = results[0]["raw_content"]
|
|
if "Just a moment" in html:
|
|
return None
|
|
return _parse_campaign_html(html, url, "tavily")
|
|
except Exception as e:
|
|
print(f"[kickstarter_fetcher] Tavily: {e}")
|
|
return None
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Stage 3: Wayback Machine
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def _stage_wayback(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
|
"""Fetch most recent Wayback Machine snapshot."""
|
|
import requests
|
|
|
|
try:
|
|
cdx_url = (
|
|
f"https://web.archive.org/web/timemap/json?"
|
|
f"url={url}&matchType=prefix&limit=5"
|
|
)
|
|
r = requests.get(cdx_url, timeout=10)
|
|
snapshots = r.json()
|
|
# Filter 403 snapshots, find first non-403
|
|
for snap in snapshots[1:]: # skip header
|
|
if len(snap) >= 5 and str(snap[4]) != "403":
|
|
ts = snap[1]
|
|
snapshot_url = f"https://web.archive.org/web/{ts}/{url}"
|
|
r = requests.get(
|
|
snapshot_url,
|
|
headers={
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
|
|
"Gecko/20100101 Firefox/128.0"
|
|
),
|
|
},
|
|
timeout=min(timeout, 30),
|
|
)
|
|
if r.status_code == 200 and len(r.text) > 1000:
|
|
return _parse_campaign_html(r.text, url, "wayback")
|
|
except Exception as e:
|
|
print(f"[kickstarter_fetcher] Wayback: {e}")
|
|
return None
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Stage 4: Google Cache
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def _stage_google_cache(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
|
"""Fetch via Google Web Cache."""
|
|
import requests
|
|
|
|
try:
|
|
cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}"
|
|
r = requests.get(
|
|
cache_url,
|
|
headers={
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
|
|
"Gecko/20100101 Firefox/128.0"
|
|
),
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
},
|
|
timeout=min(timeout, 30),
|
|
)
|
|
if r.status_code == 200 and len(r.text) > 1000:
|
|
if "before you continue" in r.text.lower() or "cookie" in r.text.lower()[:2000]:
|
|
return None
|
|
return _parse_campaign_html(r.text, url, "google_cache")
|
|
except Exception as e:
|
|
print(f"[kickstarter_fetcher] Google Cache: {e}")
|
|
return None
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Stage 5: Press Sources (Title-Only)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def _stage_press_sources(url: str, timeout: int = 10) -> Optional[KickstarterCampaign]:
|
|
"""Extract project name from URL slug."""
|
|
slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
|
|
if not slug_match:
|
|
return None
|
|
slug = slug_match.group(1)
|
|
project_name = slug.split("/")[-1].replace("-", " ").title()
|
|
|
|
return KickstarterCampaign(
|
|
url=url,
|
|
title=project_name,
|
|
source="press",
|
|
status="unknown",
|
|
description=(
|
|
f"[Indirekte Quelle — keine Live-Daten verfügbar. "
|
|
f"Projekt: {project_name}]"
|
|
),
|
|
fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# HTML Parser
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
def _parse_campaign_html(
|
|
html: str, url: str, source: str
|
|
) -> Optional[KickstarterCampaign]:
|
|
"""Parse Kickstarter campaign HTML to structured data."""
|
|
from bs4 import BeautifulSoup
|
|
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
# ── Title ───────────────────────────────────────────────
|
|
title = ""
|
|
if source in ("wayback", "google_cache"):
|
|
og = soup.find("meta", property="og:title")
|
|
if og and og.get("content"):
|
|
title = og["content"].strip()
|
|
if not title:
|
|
h1 = soup.find("h1")
|
|
if h1:
|
|
title = h1.get_text(" ", strip=True)[:200]
|
|
if not title:
|
|
for tag in soup.find_all(["h2", "h3"]):
|
|
txt = tag.get_text(" ", strip=True)
|
|
if len(txt) > 15:
|
|
title = txt[:200]
|
|
break
|
|
|
|
if not title:
|
|
title_tag = soup.find("title")
|
|
if title_tag:
|
|
title = title_tag.text.strip()
|
|
title = re.sub(
|
|
r"\s*by\s+.*?—\s*Kickstarter\s*$", "", title, flags=re.I
|
|
).strip()
|
|
if title.lower() in ("wayback machine", "just a moment..."):
|
|
title = ""
|
|
|
|
if not title:
|
|
og = soup.find("meta", property="og:title")
|
|
if og and og.get("content"):
|
|
title = og["content"].strip()
|
|
|
|
# ── JSON-LD ─────────────────────────────────────────────
|
|
pledged = ""
|
|
goal = ""
|
|
backers = 0
|
|
description = ""
|
|
image_url = ""
|
|
|
|
for script in soup.find_all("script", type="application/ld+json"):
|
|
try:
|
|
data = json.loads(script.string)
|
|
if not isinstance(data, dict):
|
|
continue
|
|
items = data.get("@graph", [data])
|
|
for item in items:
|
|
if isinstance(item, dict):
|
|
if not title and item.get("name"):
|
|
title = item["name"]
|
|
if not description and item.get("description"):
|
|
description = str(item["description"])[:500]
|
|
if item.get("image") and not image_url:
|
|
img = item["image"]
|
|
if isinstance(img, list):
|
|
img = img[0] if img else ""
|
|
if isinstance(img, str):
|
|
image_url = img
|
|
except (json.JSONDecodeError, AttributeError, TypeError):
|
|
pass
|
|
|
|
# ── Text regex ──────────────────────────────────────────
|
|
text = soup.get_text(" ", strip=True)
|
|
|
|
for pat in [
|
|
r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:pledged|von|zugesagt)",
|
|
r"([\d,.]+)\s*(?:€|EUR|USD|\$|£)\s*pledged",
|
|
]:
|
|
m = re.search(pat, text, re.I)
|
|
if m:
|
|
pledged = m.group(0).strip()
|
|
break
|
|
|
|
for pat in [
|
|
r"(?:goal|Ziel)\s*(?:of|von)?\s*(?:€|EUR|USD|\$|£)\s*([\d,.]+)",
|
|
r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:goal|Ziel)",
|
|
]:
|
|
m = re.search(pat, text, re.I)
|
|
if m:
|
|
goal = m.group(0).strip()
|
|
break
|
|
|
|
backers_match = re.search(r"([\d,]+)\s*backers?", text, re.I)
|
|
if backers_match:
|
|
try:
|
|
backers = int(backers_match.group(1).replace(",", ""))
|
|
except ValueError:
|
|
pass
|
|
|
|
days_left = -1
|
|
days_match = re.search(
|
|
r"(\d+)\s*(?:days?|Tage)\s*(?:to go|left|verbleibend)", text, re.I
|
|
)
|
|
if days_match:
|
|
try:
|
|
days_left = int(days_match.group(1))
|
|
except ValueError:
|
|
pass
|
|
|
|
# ── Status ──────────────────────────────────────────────
|
|
status = "unknown"
|
|
text_lower = text.lower()
|
|
if any(t in text_lower for t in ["successfully funded", "erfolgreich finanziert"]):
|
|
status = "funded"
|
|
elif any(t in text_lower for t in ["cancelled", "abgebrochen"]):
|
|
status = "cancelled"
|
|
elif days_left >= 0:
|
|
status = "live"
|
|
|
|
if not description:
|
|
meta_desc = soup.find("meta", attrs={"name": "description"})
|
|
if meta_desc and meta_desc.get("content"):
|
|
description = meta_desc["content"][:500]
|
|
|
|
if not title:
|
|
return None
|
|
|
|
return KickstarterCampaign(
|
|
url=url,
|
|
title=title,
|
|
pledged=pledged,
|
|
goal=goal,
|
|
backers=backers,
|
|
days_left=days_left,
|
|
status=status,
|
|
description=description,
|
|
image_url=image_url,
|
|
source=source,
|
|
raw_html=text[:30000],
|
|
fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# CLI
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python kickstarter_fetcher.py <kickstarter-url>")
|
|
print("\nStatus: Alle Stages blockiert. Nur Titel-Fallback (Stage 5) liefert Daten.")
|
|
sys.exit(1)
|
|
|
|
url = sys.argv[1]
|
|
print(f"Fetching: {url}")
|
|
print("-" * 60)
|
|
print(status_report(fetch_campaign(url)))
|