feat: Kickstarter LIVE-Daten via FlareSolverr — DURCHBRUCH!
Stage 0: FlareSolverr (Headless Chromium, localhost:8191)
- Löst Cloudflare-Challenge automatisch
- Extrahiert Campaign-Daten aus embedded Flat-JSON:
project_name, project_backers_count, project_pledged_usd,
project_goal_usd, project_state, project_deadline, project_blurb
Getestet: Earthborne Trailblazer → 49,130 / 3,969 backers / successful
Kein Proxy nötig — FlareSolverr's Chromium reicht!
Setup: FlareSolverr in ~/workspace/FlareSolverr/
Start: cd src && python start_flaresolverr.py
Patch: --headless=new statt Xvfb (utils.py)
Kickstarter-Fetcher jetzt: 2 Stages (FlareSolverr → Slug-Fallback)
This commit is contained in:
Hermes Agent
+134
-345
@@ -2,66 +2,52 @@
|
||||
"""
|
||||
Kickstarter Campaign Fetcher — Multi-Stage Fallback Strategy
|
||||
===========================================================
|
||||
Tested 2026-06-20 — Results:
|
||||
Final version 2026-06-21 — FlareSolverr breakthrough!
|
||||
|
||||
Stage 1: Playwright + iproyal Proxy → ❌ HTTP 403 (Proxy erkannt)
|
||||
Stage 2: Tavily Extract API → ❌ "Failed to fetch url"
|
||||
Stage 3: Wayback Machine → ❌ Archiviert nur 403-Seiten
|
||||
Stage 4: Google Cache → ⚠️ EU-Cookie-Wall blockiert
|
||||
Stage 5: Press Sources (URL-Slug) → ✅ Titel (keine Funding-Daten)
|
||||
Stage X: Kicktraq API → 🔮 Zu evaluieren (Drittanbieter)
|
||||
Stages:
|
||||
0. FlareSolverr (localhost:8191) → LIVE campaign data! ✅
|
||||
1. Fallback: URL-Slug → Titel (immer)
|
||||
|
||||
ACTION REQUIRED: Keine der Stages liefert Live-Funding-Daten.
|
||||
Option A: Bright Data / Oxylabs Web Unlocker (Kickstarter-spezifisch)
|
||||
Option B: Kicktraq API (https://www.kicktraq.com/api/)
|
||||
Option C: Manuell recherchieren (Autor öffnet Kampagne im Browser)
|
||||
How it works:
|
||||
FlareSolverr runs a headless Chromium that solves Cloudflare challenges.
|
||||
Kickstarter embeds campaign data as flat JSON fields in the HTML:
|
||||
project_name, project_backers_count, project_current_amount_pledged_usd, etc.
|
||||
We extract these via regex → structured KickstarterCampaign object.
|
||||
|
||||
Usage:
|
||||
from kickstarter_fetcher import fetch_campaign, status_report
|
||||
c = fetch_campaign("https://www.kickstarter.com/projects/...")
|
||||
print(status_report(c))
|
||||
|
||||
Author: Cody (Hermes Agent), 2026-06-20
|
||||
Author: Cody (Hermes Agent), 2026-06-20/21
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Configuration
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
# iproyal Residential Proxy — ERKANNT von Kickstarter (HTTP 403)
|
||||
# Nur aktivieren wenn ein besserer Proxy verfügbar ist
|
||||
PROXY_CONFIG = {
|
||||
"server": "http://8B7M8nJOfXTNHPUw:lR1VidJhN5RrRbeK_region-northamerica@geo.iproyal.com:12321",
|
||||
}
|
||||
PROXY_ENABLED = False # Auf True setzen wenn ein unblockter Proxy verfügbar ist
|
||||
|
||||
TAVILY_API_KEY = os.environ.get(
|
||||
"TAVILY_API_KEY",
|
||||
"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",
|
||||
)
|
||||
TAVILY_ENABLED = True # Wieder aktivieren wenn Tavily Kickstarter unterstützt
|
||||
FLARESOLVERR_URL = "http://localhost:8191/v1"
|
||||
FLARESOLVERR_TIMEOUT = 60 # seconds for KS page load
|
||||
|
||||
|
||||
@dataclass
|
||||
class KickstarterCampaign:
|
||||
url: str
|
||||
title: str = ""
|
||||
pledged: str = ""
|
||||
goal: str = ""
|
||||
pledged: str = "" # e.g. "$549,130"
|
||||
goal: str = "" # e.g. "$150,000"
|
||||
backers: int = 0
|
||||
days_left: int = -1
|
||||
status: str = "unknown"
|
||||
status: str = "unknown" # "live", "successful", "cancelled"
|
||||
description: str = ""
|
||||
image_url: str = ""
|
||||
source: str = ""
|
||||
raw_html: str = ""
|
||||
source: str = "" # "flaresolverr", "slug"
|
||||
fetched_at: str = ""
|
||||
|
||||
|
||||
@@ -69,372 +55,175 @@ class KickstarterCampaign:
|
||||
# Public API
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
def fetch_campaign(url: str, timeout: int = 45) -> Optional[KickstarterCampaign]:
|
||||
def fetch_campaign(url: str, timeout: int = 90) -> Optional[KickstarterCampaign]:
|
||||
"""
|
||||
Fetch a Kickstarter campaign page with multi-stage fallback.
|
||||
Returns KickstarterCampaign or None if all stages fail.
|
||||
Fetch a Kickstarter campaign page.
|
||||
Stage 0: FlareSolverr (LIVE data via headless Chromium)
|
||||
Stage 1: URL-Slug fallback (title only)
|
||||
"""
|
||||
|
||||
# Stage 1: Playwright via Residential Proxy (nur wenn aktiviert)
|
||||
if PROXY_ENABLED:
|
||||
result = _stage_playwright(url, timeout)
|
||||
# Stage 0: FlareSolverr
|
||||
result = _stage_flaresolverr(url, timeout)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# Stage 2: Tavily Extract (nur wenn aktiviert)
|
||||
if TAVILY_ENABLED:
|
||||
result = _stage_tavily(url, timeout)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# Stage 3: Wayback Machine (archiviert leider nur 403-Seiten)
|
||||
result = _stage_wayback(url, timeout)
|
||||
if result and result.title and "wayback machine" not in result.title.lower():
|
||||
return result
|
||||
|
||||
# Stage 4: Google Cache (EU-Cookie-Wall blockiert)
|
||||
result = _stage_google_cache(url, timeout)
|
||||
if result and "bevor sie zu google" not in result.title.lower():
|
||||
return result
|
||||
|
||||
# Stage 5: Press Sources (Titel aus URL-Slug)
|
||||
return _stage_press_sources(url, timeout)
|
||||
# Stage 1: Slug
|
||||
return _stage_slug(url)
|
||||
|
||||
|
||||
def status_report(campaign: Optional[KickstarterCampaign]) -> str:
|
||||
"""Generate a human-readable status report."""
|
||||
if not campaign:
|
||||
return "❌ KEINE DATEN — Alle Stages fehlgeschlagen."
|
||||
return "❌ KEINE DATEN"
|
||||
|
||||
if campaign.source == "press":
|
||||
if campaign.source == "slug":
|
||||
return (
|
||||
f"⚠️ NUR TITEL (Presse-Fallback)\n"
|
||||
f"⚠️ NUR TITEL\n"
|
||||
f" {campaign.title}\n"
|
||||
f" Keine Live-Daten verfügbar.\n"
|
||||
f" → Bitte manuell recherchieren: {campaign.url}"
|
||||
)
|
||||
|
||||
return (
|
||||
f"✅ {campaign.source.upper()}\n"
|
||||
f"✅ LIVE ({campaign.source})\n"
|
||||
f" {campaign.title}\n"
|
||||
f" {campaign.pledged} von {campaign.goal}\n"
|
||||
f" {campaign.backers} Unterstützer\n"
|
||||
f" {campaign.days_left} Tage verbleibend\n"
|
||||
f" {campaign.backers:,} Unterstützer\n"
|
||||
f" Status: {campaign.status}"
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Stage 1: Playwright Headless Browser via Proxy
|
||||
# Stage 0: FlareSolverr → Headless Chromium → KS Page → Parse
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
def _stage_playwright(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
||||
"""Fetch via headless Chromium with Residential Proxy."""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(
|
||||
headless=True,
|
||||
args=[
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
],
|
||||
)
|
||||
ctx_kwargs = {
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
),
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
}
|
||||
if PROXY_ENABLED:
|
||||
ctx_kwargs["proxy"] = PROXY_CONFIG
|
||||
|
||||
context = browser.new_context(**ctx_kwargs)
|
||||
page = context.new_page()
|
||||
page.goto(url, timeout=timeout * 1000, wait_until="domcontentloaded")
|
||||
page.wait_for_timeout(8000)
|
||||
html = page.content()
|
||||
browser.close()
|
||||
|
||||
if "Just a moment" in html:
|
||||
return None
|
||||
return _parse_campaign_html(html, url, "playwright")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[kickstarter_fetcher] Playwright: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Stage 2: Tavily Extract API
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
def _stage_tavily(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
||||
"""Fetch via Tavily Extract API."""
|
||||
def _stage_flaresolverr(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
||||
"""
|
||||
Fetch Kickstarter page via FlareSolverr (headless Chromium).
|
||||
Extracts campaign data from embedded flat JSON fields.
|
||||
"""
|
||||
import requests
|
||||
|
||||
try:
|
||||
r = requests.post(
|
||||
"https://api.tavily.com/extract",
|
||||
FLARESOLVERR_URL,
|
||||
json={
|
||||
"api_key": TAVILY_API_KEY,
|
||||
"urls": [url],
|
||||
"extract_depth": "advanced",
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": min(timeout, FLARESOLVERR_TIMEOUT) * 1000,
|
||||
},
|
||||
timeout=min(timeout, 30),
|
||||
timeout=timeout + 10,
|
||||
)
|
||||
data = r.json()
|
||||
results = data.get("results", [])
|
||||
if results and results[0].get("raw_content"):
|
||||
html = results[0]["raw_content"]
|
||||
if "Just a moment" in html:
|
||||
return None
|
||||
return _parse_campaign_html(html, url, "tavily")
|
||||
except Exception as e:
|
||||
print(f"[kickstarter_fetcher] Tavily: {e}")
|
||||
print(f"[kickstarter_fetcher] FlareSolverr request error: {e}")
|
||||
return None
|
||||
|
||||
if data.get("status") != "ok":
|
||||
print(f"[kickstarter_fetcher] FlareSolverr error: {data.get('message', '?')}")
|
||||
return None
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Stage 3: Wayback Machine
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
sol = data.get("solution", {})
|
||||
html = sol.get("response", "")
|
||||
if not html or len(html) < 5000:
|
||||
print(f"[kickstarter_fetcher] Empty/short response: {len(html)} bytes")
|
||||
return None
|
||||
|
||||
def _stage_wayback(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
||||
"""Fetch most recent Wayback Machine snapshot."""
|
||||
import requests
|
||||
# Check for Cloudflare block in rendered page
|
||||
if "Just a moment" in html[:2000]:
|
||||
print("[kickstarter_fetcher] Cloudflare block after rendering")
|
||||
return None
|
||||
|
||||
return _parse_campaign_native(html, url, "flaresolverr")
|
||||
|
||||
|
||||
def _parse_campaign_native(
|
||||
html: str, url: str, source: str
|
||||
) -> Optional[KickstarterCampaign]:
|
||||
"""
|
||||
Parse Kickstarter campaign data from embedded flat JSON fields.
|
||||
Kickstarter injects data like:
|
||||
"project_name":"Earthborne Trailblazer"
|
||||
"project_backers_count":3969
|
||||
"project_current_amount_pledged_usd":549130.0
|
||||
"project_goal_usd":150000.0
|
||||
"project_state":"successful"
|
||||
"project_deadline":"2026-06-16T11:01:05-04:00"
|
||||
"project_blurb":"Explore the wilderness..."
|
||||
"""
|
||||
|
||||
def _re_first(pattern: str, html: str, group: int = 1) -> Optional[str]:
|
||||
m = re.search(pattern, html)
|
||||
return m.group(group) if m else None
|
||||
|
||||
def _re_int(pattern: str, html: str) -> int:
|
||||
m = re.search(pattern, html)
|
||||
return int(m.group(1)) if m else 0
|
||||
|
||||
def _re_float(pattern: str, html: str) -> float:
|
||||
m = re.search(pattern, html)
|
||||
return float(m.group(1)) if m else 0.0
|
||||
|
||||
# Extract all fields
|
||||
name = _re_first(r'"project_name":"([^"]+)"', html)
|
||||
if not name:
|
||||
# Try og:title as fallback
|
||||
name = _re_first(r'<meta\s+property="og:title"\s+content="([^"]+)"', html)
|
||||
if not name:
|
||||
return None
|
||||
|
||||
blurb = _re_first(r'"project_blurb":"([^"]*)"', html)
|
||||
currency = _re_first(r'"project_currency":"([^"]+)"', html) or "USD"
|
||||
state = _re_first(r'"project_state":"([^"]+)"', html) or "unknown"
|
||||
|
||||
pledged_usd = _re_float(r'"project_current_amount_pledged_usd":([\d.]+)', html)
|
||||
goal_usd = _re_float(r'"project_goal_usd":([\d.]+)', html)
|
||||
backers = _re_int(r'"project_backers_count":(\d+)', html)
|
||||
deadline = _re_first(r'"project_deadline":"([^"]+)"', html)
|
||||
|
||||
# Format amounts
|
||||
def fmt_amount(usd_val: float) -> str:
|
||||
return f"${usd_val:,.0f}"
|
||||
|
||||
# Calculate days left
|
||||
days_left = -1
|
||||
if deadline:
|
||||
try:
|
||||
cdx_url = (
|
||||
f"https://web.archive.org/web/timemap/json?"
|
||||
f"url={url}&matchType=prefix&limit=5"
|
||||
)
|
||||
r = requests.get(cdx_url, timeout=10)
|
||||
snapshots = r.json()
|
||||
# Filter 403 snapshots, find first non-403
|
||||
for snap in snapshots[1:]: # skip header
|
||||
if len(snap) >= 5 and str(snap[4]) != "403":
|
||||
ts = snap[1]
|
||||
snapshot_url = f"https://web.archive.org/web/{ts}/{url}"
|
||||
r = requests.get(
|
||||
snapshot_url,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
|
||||
"Gecko/20100101 Firefox/128.0"
|
||||
),
|
||||
},
|
||||
timeout=min(timeout, 30),
|
||||
)
|
||||
if r.status_code == 200 and len(r.text) > 1000:
|
||||
return _parse_campaign_html(r.text, url, "wayback")
|
||||
except Exception as e:
|
||||
print(f"[kickstarter_fetcher] Wayback: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Stage 4: Google Cache
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
def _stage_google_cache(url: str, timeout: int) -> Optional[KickstarterCampaign]:
|
||||
"""Fetch via Google Web Cache."""
|
||||
import requests
|
||||
|
||||
try:
|
||||
cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}"
|
||||
r = requests.get(
|
||||
cache_url,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
|
||||
"Gecko/20100101 Firefox/128.0"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
timeout=min(timeout, 30),
|
||||
)
|
||||
if r.status_code == 200 and len(r.text) > 1000:
|
||||
if "before you continue" in r.text.lower() or "cookie" in r.text.lower()[:2000]:
|
||||
return None
|
||||
return _parse_campaign_html(r.text, url, "google_cache")
|
||||
except Exception as e:
|
||||
print(f"[kickstarter_fetcher] Google Cache: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Stage 5: Press Sources (Title-Only)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
def _stage_press_sources(url: str, timeout: int = 10) -> Optional[KickstarterCampaign]:
|
||||
"""Extract project name from URL slug."""
|
||||
slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
|
||||
if not slug_match:
|
||||
return None
|
||||
slug = slug_match.group(1)
|
||||
project_name = slug.split("/")[-1].replace("-", " ").title()
|
||||
from datetime import datetime
|
||||
dt = datetime.fromisoformat(deadline.replace("Z", "+00:00"))
|
||||
delta = dt - datetime.now(dt.tzinfo)
|
||||
days_left = max(0, delta.days)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return KickstarterCampaign(
|
||||
url=url,
|
||||
title=project_name,
|
||||
source="press",
|
||||
status="unknown",
|
||||
description=(
|
||||
f"[Indirekte Quelle — keine Live-Daten verfügbar. "
|
||||
f"Projekt: {project_name}]"
|
||||
),
|
||||
title=name,
|
||||
pledged=fmt_amount(pledged_usd),
|
||||
goal=fmt_amount(goal_usd),
|
||||
backers=backers,
|
||||
days_left=days_left if state == "live" else -1,
|
||||
status=state,
|
||||
description=(blurb or "")[:500],
|
||||
source=source,
|
||||
fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# HTML Parser
|
||||
# Stage 1: URL-Slug Fallback
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
def _parse_campaign_html(
|
||||
html: str, url: str, source: str
|
||||
) -> Optional[KickstarterCampaign]:
|
||||
"""Parse Kickstarter campaign HTML to structured data."""
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# ── Title ───────────────────────────────────────────────
|
||||
title = ""
|
||||
if source in ("wayback", "google_cache"):
|
||||
og = soup.find("meta", property="og:title")
|
||||
if og and og.get("content"):
|
||||
title = og["content"].strip()
|
||||
if not title:
|
||||
h1 = soup.find("h1")
|
||||
if h1:
|
||||
title = h1.get_text(" ", strip=True)[:200]
|
||||
if not title:
|
||||
for tag in soup.find_all(["h2", "h3"]):
|
||||
txt = tag.get_text(" ", strip=True)
|
||||
if len(txt) > 15:
|
||||
title = txt[:200]
|
||||
break
|
||||
|
||||
if not title:
|
||||
title_tag = soup.find("title")
|
||||
if title_tag:
|
||||
title = title_tag.text.strip()
|
||||
title = re.sub(
|
||||
r"\s*by\s+.*?—\s*Kickstarter\s*$", "", title, flags=re.I
|
||||
).strip()
|
||||
if title.lower() in ("wayback machine", "just a moment..."):
|
||||
title = ""
|
||||
|
||||
if not title:
|
||||
og = soup.find("meta", property="og:title")
|
||||
if og and og.get("content"):
|
||||
title = og["content"].strip()
|
||||
|
||||
# ── JSON-LD ─────────────────────────────────────────────
|
||||
pledged = ""
|
||||
goal = ""
|
||||
backers = 0
|
||||
description = ""
|
||||
image_url = ""
|
||||
|
||||
for script in soup.find_all("script", type="application/ld+json"):
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
items = data.get("@graph", [data])
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
if not title and item.get("name"):
|
||||
title = item["name"]
|
||||
if not description and item.get("description"):
|
||||
description = str(item["description"])[:500]
|
||||
if item.get("image") and not image_url:
|
||||
img = item["image"]
|
||||
if isinstance(img, list):
|
||||
img = img[0] if img else ""
|
||||
if isinstance(img, str):
|
||||
image_url = img
|
||||
except (json.JSONDecodeError, AttributeError, TypeError):
|
||||
pass
|
||||
|
||||
# ── Text regex ──────────────────────────────────────────
|
||||
text = soup.get_text(" ", strip=True)
|
||||
|
||||
for pat in [
|
||||
r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:pledged|von|zugesagt)",
|
||||
r"([\d,.]+)\s*(?:€|EUR|USD|\$|£)\s*pledged",
|
||||
]:
|
||||
m = re.search(pat, text, re.I)
|
||||
if m:
|
||||
pledged = m.group(0).strip()
|
||||
break
|
||||
|
||||
for pat in [
|
||||
r"(?:goal|Ziel)\s*(?:of|von)?\s*(?:€|EUR|USD|\$|£)\s*([\d,.]+)",
|
||||
r"(?:€|EUR|USD|\$|£)\s*([\d,.]+)\s*(?:goal|Ziel)",
|
||||
]:
|
||||
m = re.search(pat, text, re.I)
|
||||
if m:
|
||||
goal = m.group(0).strip()
|
||||
break
|
||||
|
||||
backers_match = re.search(r"([\d,]+)\s*backers?", text, re.I)
|
||||
if backers_match:
|
||||
try:
|
||||
backers = int(backers_match.group(1).replace(",", ""))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
days_left = -1
|
||||
days_match = re.search(
|
||||
r"(\d+)\s*(?:days?|Tage)\s*(?:to go|left|verbleibend)", text, re.I
|
||||
)
|
||||
if days_match:
|
||||
try:
|
||||
days_left = int(days_match.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# ── Status ──────────────────────────────────────────────
|
||||
status = "unknown"
|
||||
text_lower = text.lower()
|
||||
if any(t in text_lower for t in ["successfully funded", "erfolgreich finanziert"]):
|
||||
status = "funded"
|
||||
elif any(t in text_lower for t in ["cancelled", "abgebrochen"]):
|
||||
status = "cancelled"
|
||||
elif days_left >= 0:
|
||||
status = "live"
|
||||
|
||||
if not description:
|
||||
meta_desc = soup.find("meta", attrs={"name": "description"})
|
||||
if meta_desc and meta_desc.get("content"):
|
||||
description = meta_desc["content"][:500]
|
||||
|
||||
if not title:
|
||||
return None
|
||||
def _stage_slug(url: str) -> KickstarterCampaign:
|
||||
"""Extract project name from URL slug."""
|
||||
slug_match = re.search(r"projects/([^/]+/[^/?#]+)", url)
|
||||
slug = slug_match.group(1) if slug_match else url.rstrip("/").split("/")[-1]
|
||||
project_name = slug.split("/")[-1].replace("-", " ").title()
|
||||
|
||||
return KickstarterCampaign(
|
||||
url=url,
|
||||
title=title,
|
||||
pledged=pledged,
|
||||
goal=goal,
|
||||
backers=backers,
|
||||
days_left=days_left,
|
||||
status=status,
|
||||
description=description,
|
||||
image_url=image_url,
|
||||
source=source,
|
||||
raw_html=text[:30000],
|
||||
title=project_name,
|
||||
source="slug",
|
||||
status="unknown",
|
||||
fetched_at=time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
)
|
||||
|
||||
@@ -448,10 +237,10 @@ if __name__ == "__main__":
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python kickstarter_fetcher.py <kickstarter-url>")
|
||||
print("\nStatus: Alle Stages blockiert. Nur Titel-Fallback (Stage 5) liefert Daten.")
|
||||
print("\nRequires FlareSolverr running on localhost:8191")
|
||||
sys.exit(1)
|
||||
|
||||
url = sys.argv[1]
|
||||
print(f"Fetching: {url}")
|
||||
print("-" * 60)
|
||||
print("-" * 50)
|
||||
print(status_report(fetch_campaign(url)))
|
||||
|
||||
Reference in New Issue
Block a user