#!/usr/bin/env python3
"""Fetch Kickstarter Discover API pages 1-5 via FlareSolverr, extract & filter projects."""
import urllib.request, json, sys, os, re
from datetime import datetime, timezone

FLARESOLVERR = "http://localhost:8191/v1"
SKIP_KW = ['5e', '5th edition', 'handbook', 'rpg', 'd&d', 'dnd', 'roleplaying game', 'ttrpg',
           'dungeon master', 'stl', '15mm', '28mm', '32mm', 'resin', 'miniature',
           'plastic regiment', 'multi-part', '3d print', 'terrain',
           'dice bag', 'dice tray', 'playmat', 'map pack', 'battle map',
           'touchscreen', 'mapcase', 'character sheet', 'spellbook',
           'soundtrack', 'soundscapes', 'music album', 'props', 'makeup',
           'accessory']

def get_ks(url, session=None):
    payload = {"cmd": "request.get", "url": url, "maxTimeout": 45000}
    if session:
        payload["session"] = session
    req = urllib.request.Request(FLARESOLVERR,
        data=json.dumps(payload).encode(),
        headers={"Content-Type": "application/json"}, method="POST")
    with urllib.request.urlopen(req, timeout=60) as resp:
        return json.loads(resp.read().decode())

all_projects = []
session = None

for page in range(1, 6):
    url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page}&format=json"
    print(f"Fetching KS page {page}...", flush=True)
    
    try:
        result = get_ks(url, session=session)
        if result.get("status") != "ok":
            print(f"  ERROR: {result.get('message', 'Unknown error')}", flush=True)
            continue
        
        # Save session for subsequent requests
        session = result.get("solution", {}).get("session")
        
        resp_text = result["solution"]["response"]
        
        # Extract JSON from HTML-wrapped response
        if "<pre>" in resp_text:
            match = re.search(r'<pre>\s*(.+?)\s*</pre>', resp_text, re.DOTALL)
            if match:
                raw_json = match.group(1)
            else:
                print(f"  Could not extract JSON from <pre>", flush=True)
                continue
        else:
            raw_json = resp_text
        
        inner = json.loads(raw_json)
        projects = inner.get("projects", [])
        
        for p in projects:
            name = (p.get("name") or "").lower()
            blurb = (p.get("blurb") or "").lower()
            combined = name + " " + blurb
            is_skip = any(kw in combined for kw in SKIP_KW)
            if is_skip:
                continue
            
            # Extract fields
            slug = p.get("slug", "")
            project_id = p.get("id")
            
            # Deadline handling
            deadline_raw = p.get("deadline")
            if isinstance(deadline_raw, int):
                deadline_dt = datetime.fromtimestamp(deadline_raw, tz=timezone.utc)
            elif isinstance(deadline_raw, str):
                try:
                    deadline_dt = datetime.fromisoformat(deadline_raw.replace('Z', '+00:00'))
                except:
                    deadline_dt = None
            else:
                deadline_dt = None
            
            # State check
            state = p.get("state", "live")
            
            all_projects.append({
                "id": project_id,
                "name": p.get("name", ""),
                "slug": slug,
                "url": f"https://www.kickstarter.com/projects/{slug}" if slug else "",
                "blurb": p.get("blurb", ""),
                "pledged": p.get("pledged", 0),
                "goal": p.get("goal", 0),
                "backers_count": p.get("backers_count", 0),
                "currency": p.get("currency", "USD"),
                "deadline": deadline_dt.isoformat() if deadline_dt else None,
                "deadline_ts": p.get("deadline"),
                "photo": (p.get("photo") or {}).get("1024x768", ""),
                "state": state,
                "creator": (p.get("creator") or {}).get("name", ""),
                "location": (p.get("location") or {}).get("short", ""),
            })
        
        print(f"  Found {len(projects)} raw → {sum(1 for p in projects if not any(kw in ((p.get('name') or '') + ' ' + (p.get('blurb') or '')).lower() for kw in SKIP_KW))} filtered (page total: {len(all_projects)})", flush=True)
    
    except Exception as e:
        print(f"  EXCEPTION: {e}", flush=True)
        continue

print(f"\nTotal KS board games: {len(all_projects)}", flush=True)

with open("/home/hermes/workspace/ks_filtered.json", "w") as f:
    json.dump(all_projects, f, indent=2, ensure_ascii=False)

print("SAVED to ks_filtered.json", flush=True)