BSN-Chatsystem/fetch_ks_kw26.py

#!/usr/bin/env python3
"""
Final: Fetch ALL KW26 data and save comprehensive JSON.
Combines known article campaigns + additional KW26-ending campaigns from KS discover API.
"""

import sys
import json
import time
import re
from datetime import datetime, timezone

sys.path.insert(0, "/home/hermes/workspace/bsn-chatbot")
from kickstarter_fetcher import fetch_campaign, status_report, FLARESOLVERR_URL
import requests

# ── Known campaigns from the article ──────────────────────────
CAMPAIGNS = [
    ("Cult of the Lamb", "https://www.kickstarter.com/projects/paperfortgames/cult-of-the-lamb-the-board-game"),
    ("Monolith", "https://www.kickstarter.com/projects/plaidhatgames/monolith-1"),
    ("Yumegari", "https://www.kickstarter.com/projects/lucky-duck-games/yumegari"),
    ("Defenders of Hogwarts", "https://www.kickstarter.com/projects/minalima/defenders-of-hogwarts"),
    ("CARTA IMPERIA", "https://www.kickstarter.com/projects/guntowergames/carta-imperia"),
    ("Headliner: Encore", "https://www.kickstarter.com/projects/chriscouch/headliner-encore-campaign"),
    ("OrganATTACK: Below the Belt", "https://www.kickstarter.com/projects/theawkwardyeti/organattack-below-the-belt-edition"),
    ("Fictitious", "https://www.kickstarter.com/projects/11017881/fictitious-a-mythical-card-game-of-creature-crafting"),
]


def fetch_discover_kw26():
    """Fetch all KW26-ending tabletop game campaigns from KS discover API."""
    kw26_start = datetime(2026, 6, 23, tzinfo=timezone.utc)
    kw26_end = datetime(2026, 6, 29, 23, 59, 59, tzinfo=timezone.utc)

    all_projects = []

    for page in range(1, 8):
        r = requests.post(
            FLARESOLVERR_URL,
            json={
                "cmd": "request.get",
                "url": f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page}&format=json",
                "maxTimeout": 60000,
            },
            timeout=90,
        )
        data = r.json()
        html = data.get("solution", {}).get("response", "")

        json_match = re.search(r"<pre>(.*?)</pre>", html, re.DOTALL)
        if not json_match:
            break

        discover = json.loads(json_match.group(1))
        projects = discover.get("projects", [])
        if not projects:
            break

        for p in projects:
            deadline_ts = p.get("deadline", 0)
            if not deadline_ts:
                continue
            dt = datetime.fromtimestamp(deadline_ts, tz=timezone.utc)
            if kw26_start <= dt <= kw26_end:
                all_projects.append(p)

        time.sleep(1)

    return all_projects


def campaign_to_dict(c, list_name="", extra=None):
    """Convert KickstarterCampaign to a JSON-serializable dict."""
    if c is None:
        base = {"error": "fetch failed"}
        if extra:
            base.update(extra)
        base["list_name"] = list_name
        return base

    pct = None
    if c.pledged and c.goal:
        try:
            pledged_num = float(c.pledged.replace("$", "").replace(",", "").replace("€", "").replace("£", ""))
            goal_num = float(c.goal.replace("$", "").replace(",", "").replace("€", "").replace("£", ""))
            if goal_num > 0:
                pct = round(pledged_num / goal_num * 100, 1)
        except:
            pass

    d = {
        "name": c.title,
        "url": c.url,
        "pledged": c.pledged,
        "goal": c.goal,
        "percent_funded": pct,
        "backers": c.backers,
        "days_left": c.days_left,
        "status": c.status,
        "description": (c.description or "")[:200],
        "source": c.source,
        "fetched_at": c.fetched_at,
        "list_name": list_name,
    }
    if extra:
        d.update(extra)
    return d


def main():
    results = []

    print("=" * 60)
    print("KW26 Kickstarter Board Game Campaigns — LIVE Data")
    print("=" * 60)

    # ── Step 1: Fetch known campaigns ──
    print("\n── 8 Known Campaigns from Article ──")
    for name, url in CAMPAIGNS:
        print(f"\n  [{name}] {url}")
        c = fetch_campaign(url, timeout=90)
        print(f"  {status_report(c)}")
        results.append(campaign_to_dict(c, list_name=name))
        time.sleep(2)

    # ── Step 2: Fetch additional KW26 campaigns from discover API ──
    print("\n── Searching KS Discover API for KW26 Tabletop Games ──")
    additional = fetch_discover_kw26()
    print(f"  Found {len(additional)} additional KW26 campaigns")

    known_urls = {url for _, url in CAMPAIGNS}
    for p in additional:
        url = p.get("urls", {}).get("web", {}).get("project", "")
        if not url or url in known_urls:
            continue

        name = p.get("name", "?")
        currency = p.get("currency", "USD")
        pledged = p.get("pledged", 0)
        goal = p.get("goal", 0)
        backers = p.get("backers_count", 0)
        deadline_ts = p.get("deadline", 0)
        state = p.get("state", "?")
        blurb = p.get("blurb", "")

        dt = datetime.fromtimestamp(deadline_ts, tz=timezone.utc)
        days_left = max(0, (dt - datetime.now(timezone.utc)).days)
        end_date = dt.strftime("%Y-%m-%d")
        pct = round(pledged / goal * 100, 1) if goal > 0 else 0

        # Format amounts with currency
        if currency == "EUR":
            pledged_str = f"€{pledged:,.0f}"
            goal_str = f"€{goal:,.0f}"
        elif currency == "GBP":
            pledged_str = f"£{pledged:,.0f}"
            goal_str = f"£{goal:,.0f}"
        elif currency == "MXN":
            pledged_str = f"MX${pledged:,.0f}"
            goal_str = f"MX${goal:,.0f}"
        else:
            pledged_str = f"${pledged:,.0f}"
            goal_str = f"${goal:,.0f}"

        print(f"\n  [{name[:60]}] {url}")
        print(f"  {pledged_str} / {goal_str} ({pct}%) | {backers} backers | {days_left}d left | ends {end_date} | {state}")

        results.append({
            "name": name,
            "url": url,
            "pledged": pledged_str,
            "goal": goal_str,
            "percent_funded": pct,
            "backers": backers,
            "days_left": days_left,
            "end_date": end_date,
            "status": state,
            "description": (blurb or "")[:200],
            "source": "discover_api",
            "fetched_at": time.strftime("%Y-%m-%d %H:%M:%S"),
            "list_name": "additional_kw26",
        })

    # ── Step 3: Save ──
    output_path = "/home/hermes/workspace/ks_daten_kw26.json"
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(results, f, indent=2, ensure_ascii=False)

    print(f"\n{'=' * 60}")
    print(f"Saved {len(results)} campaigns to {output_path}")

    # ── Summary table ──
    print(f"\n{'─' * 90}")
    print(f"{'Name':<38} {'Pledged':>14} {'Goal':>14} {'%':>8} {'Backers':>8} {'Days':>5} {'Status':>12}")
    print(f"{'─' * 90}")
    for r in results:
        name = (r.get("list_name") or r.get("name", ""))[:37]
        pledged = r.get("pledged", "?")
        goal = r.get("goal", "?")
        pct = f"{r['percent_funded']}%" if r.get("percent_funded") is not None else "?"
        backers = f"{r.get('backers', 0):,}" if r.get("backers") else "?"
        days = str(r.get("days_left", "?"))
        status = r.get("status", "?")
        print(f"{name:<38} {pledged:>14} {goal:>14} {pct:>8} {backers:>8} {days:>5} {status:<12}")
    print(f"{'─' * 90}")


if __name__ == "__main__":
    main()