BSN-Chatsystem/collect_ks_all.py

#!/usr/bin/env python3
"""
Collect ALL active Kickstarter board game campaigns (category_id=34).
Uses Discover API via FlareSolverr + fetch_campaign() for top-20 live data.
"""

import sys
import os
import json
import re
import time
import requests

# Add workspace to path for kickstarter_fetcher import
sys.path.insert(0, '/home/hermes/workspace/bsn-chatbot')
from kickstarter_fetcher import fetch_campaign, status_report

FLARESOLVERR_URL = "http://localhost:8191/v1"
OUTPUT_FILE = "/home/hermes/workspace/ks_all_kw27.json"

# Keywords to filter OUT (non-board-game campaigns)
EXCLUDE_KEYWORDS = [
    "stl", "3d print", "3d printable", "miniature", "miniatures",
    "rpg book", "rpg supplement", "rpg zine", "rpg tracker",
    "dice", "dice set", "dice tower",
    "playing cards", "tarot", "poker deck",
    "ttrpg", "5e", "d&d", "dungeons & dragons", "pathfinder",
    "terrain", "scenery",
]

# Keywords that suggest it IS a board game
BOARD_GAME_KEYWORDS = [
    "board game", "card game", "deck-building", "deck building",
    "worker placement", "euro", "strategy game", "family game",
    "cooperative game", "co-op game", "tile", "dice game",
    "wargame", "war game", "miniatures game", "skirmish",
    "tabletop game", "party game", "roll & write", "roll and write",
    "drafting", "area control", "engine builder", "engine building",
    "legacy", "campaign game", "adventure game",
]

def fetch_discover_page(page_num):
    """Fetch one page of the Kickstarter Discover API via FlareSolverr."""
    url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&format=json&page={page_num}"
    
    try:
        r = requests.post(
            FLARESOLVERR_URL,
            json={
                "cmd": "request.get",
                "url": url,
                "maxTimeout": 60000,
            },
            timeout=90,
        )
        data = r.json()
        if data.get("status") != "ok":
            print(f"  Page {page_num}: FlareSolverr error: {data.get('message', '?')}")
            return []
        
        html = data["solution"]["response"]
        m = re.search(r'<pre>(.*?)</pre>', html, re.DOTALL)
        if not m:
            print(f"  Page {page_num}: No JSON found in response")
            return []
        
        d = json.loads(m.group(1))
        projects = d.get("projects", [])
        print(f"  Page {page_num}: {len(projects)} projects (total hits: {d.get('total_hits')})")
        return projects
    except Exception as e:
        print(f"  Page {page_num}: Exception: {e}")
        return []

def is_board_game(project):
    """Heuristic filter: is this actually a board game?"""
    name = (project.get("name") or "").lower()
    blurb = (project.get("blurb") or "").lower()
    combined = name + " " + blurb
    
    # Check exclusions first
    for kw in EXCLUDE_KEYWORDS:
        if kw in combined:
            # But if it also has strong board game keywords, keep it
            for bg_kw in BOARD_GAME_KEYWORDS:
                if bg_kw in combined:
                    break
            else:
                return False
    
    return True

def parse_project(project):
    """Convert Discover API project to our format. Converts to USD."""
    urls = project.get("urls", {}).get("web", {})
    proj_url = urls.get("project", "")
    
    # Parse deadline
    deadline_ts = project.get("deadline", 0)
    deadline_str = ""
    days_left = -1
    if deadline_ts:
        from datetime import datetime, timezone
        dt = datetime.fromtimestamp(deadline_ts, tz=timezone.utc)
        deadline_str = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
        delta = dt - datetime.now(timezone.utc)
        days_left = max(0, delta.days)
    
    # Convert to USD: Discover API returns amounts in local currency
    # static_usd_rate converts local currency → USD
    local_pledged = project.get("pledged", 0) or 0
    local_goal = project.get("goal", 0) or 0
    usd_rate = project.get("static_usd_rate", 1.0) or 1.0
    
    pledged_usd = round(local_pledged * usd_rate, 2)
    goal_usd = round(local_goal * usd_rate, 2)
    pct = round(pledged_usd / goal_usd * 100, 1) if goal_usd > 0 else 0
    
    return {
        "name": project.get("name", ""),
        "url": proj_url,
        "pledged_usd": pledged_usd,
        "pledged_display": f"${pledged_usd:,.0f}",
        "goal_usd": goal_usd,
        "goal_display": f"${goal_usd:,.0f}",
        "percent_funded": pct,
        "backers": project.get("backers_count", 0),
        "days_left": days_left,
        "deadline": deadline_str,
        "blurb": project.get("blurb", ""),
        "country": project.get("country", ""),
        "currency": project.get("currency", ""),
        "state": project.get("state", "live"),
        "source": "discover_api",
    }

def main():
    print("=" * 60)
    print("KS ALL Campaign Collector - KW27")
    print("=" * 60)
    
    # Step 1: Fetch all pages
    print("\n[1] Fetching all Discover API pages...")
    all_projects = []
    page = 1
    while True:
        projects = fetch_discover_page(page)
        if not projects:
            break
        all_projects.extend(projects)
        page += 1
        time.sleep(1)  # Be gentle
    
    print(f"\nTotal raw projects fetched: {len(all_projects)}")
    
    # Step 2: Filter for board games
    print("\n[2] Filtering for board games...")
    board_games = []
    excluded = []
    for p in all_projects:
        if is_board_game(p):
            board_games.append(p)
        else:
            excluded.append(p)
    
    print(f"  Board games: {len(board_games)}")
    print(f"  Excluded: {len(excluded)}")
    if excluded:
        print(f"  Excluded examples: {[e.get('name','')[:60] for e in excluded[:10]]}")
    
    # Step 3: Parse all board games
    print("\n[3] Parsing board game data...")
    parsed = [parse_project(p) for p in board_games]
    
    # Sort by pledged amount descending
    parsed.sort(key=lambda x: x["pledged_usd"], reverse=True)
    
    # Step 4: Fetch top-20 with FlareSolverr for LIVE data
    print("\n[4] Fetching top-20 campaigns with FlareSolverr (LIVE data)...")
    top_n = min(20, len(parsed))
    for i in range(top_n):
        p = parsed[i]
        url = p["url"]
        print(f"  [{i+1}/{top_n}] {p['name'][:60]}...")
        campaign = fetch_campaign(url, timeout=90)
        if campaign and campaign.source == "flaresolverr":
            # Update with live data
            p["pledged_display"] = campaign.pledged
            p["goal_display"] = campaign.goal
            p["backers"] = campaign.backers
            p["days_left"] = campaign.days_left
            p["status"] = campaign.status
            p["source"] = "flaresolverr_live"
            p["description"] = campaign.description
            # Also update numeric USD values from display strings
            try:
                p["pledged_usd"] = float(campaign.pledged.replace("$","").replace(",",""))
            except: pass
            try:
                p["goal_usd"] = float(campaign.goal.replace("$","").replace(",",""))
            except: pass
            # Recalculate percent
            if p["goal_usd"] > 0:
                p["percent_funded"] = round(p["pledged_usd"] / p["goal_usd"] * 100, 1)
            print(f"    ✅ LIVE: {campaign.pledged} / {campaign.goal} | {campaign.backers} backers | {campaign.days_left}d left")
        else:
            print(f"    ⚠️  FlareSolverr failed, keeping Discover API data")
        time.sleep(2)  # Rate limit
    
    # Step 5: Save
    print(f"\n[5] Saving {len(parsed)} campaigns to {OUTPUT_FILE}...")
    
    output = {
        "collected_at": time.strftime("%Y-%m-%d %H:%M:%S"),
        "total_board_games": len(parsed),
        "top20_live_fetched": top_n,
        "campaigns": parsed,
    }
    
    with open(OUTPUT_FILE, "w") as f:
        json.dump(output, f, indent=2, ensure_ascii=False)
    
    print(f"\n✅ DONE! {len(parsed)} board game campaigns saved.")
    print(f"   Top 5 by funding:")
    for p in parsed[:5]:
        print(f"   {p['pledged_display']} / {p['goal_display']} ({p['percent_funded']}%) | {p['name'][:70]}")

if __name__ == "__main__":
    main()