#!/usr/bin/env python3
"""Fetch ALL Kickstarter board game campaigns via solver.
Handles both data-project JSON and HTML-only project cards."""
import json, re, html as html_mod, time, urllib.request
from datetime import datetime, timezone

SOLVER_URL = "http://localhost:8191/v1"
OUTPUT_FILE = "/home/hermes/workspace/ks_all_kw27_full.json"

def fetch_page(page_num):
    """Fetch one page of KS discover via solver."""
    url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page_num}"
    payload = json.dumps({
        "cmd": "request.get",
        "url": url,
        "maxTimeout": 60000
    }).encode()
    
    req = urllib.request.Request(SOLVER_URL, data=payload, headers={"Content-Type": "application/json"})
    resp = urllib.request.urlopen(req, timeout=90)
    data = json.loads(resp.read())
    
    if data.get("status") != "ok":
        raise Exception(f"Solver error: {data}")
    
    return data["solution"]["response"]

def extract_from_json(html_content):
    """Extract projects from data-project JSON attributes."""
    data_proj = re.findall(r'data-project="({.*?})"', html_content)
    projects = []
    for dp in data_proj:
        decoded = html_mod.unescape(dp)
        try:
            proj = json.loads(decoded)
            projects.append(proj)
        except Exception:
            try:
                decoded2 = html_mod.unescape(decoded)
                proj = json.loads(decoded2)
                projects.append(proj)
            except Exception as e2:
                print(f"  JSON parse error: {e2}")
    return projects

def extract_from_html(html_content):
    """Extract projects from HTML project cards (fallback when no JSON)."""
    cards = re.findall(
        r'<div[^>]*project-card-root[^>]*>(.*?)</div>\s*</div>\s*</div>\s*</div>\s*</div>',
        html_content, re.DOTALL
    )
    
    projects = []
    for block in cards:
        proj = {}
        
        # Title
        title_m = re.search(r'project-card__title[^>]*>(.*?)</a>', block, re.DOTALL)
        if title_m:
            proj["name"] = html_mod.unescape(title_m.group(1).strip())
        
        # URL
        url_m = re.search(r'href="(https?://www\.kickstarter\.com/projects/[^"?]+)', block)
        if url_m:
            proj["url"] = url_m.group(1)
        
        # Creator
        creator_m = re.search(r'project-card__creator[^>]*>.*?<span[^>]*>(.*?)</span>', block, re.DOTALL)
        if creator_m:
            proj["creator"] = creator_m.group(1).strip()
        
        # Days left / percent funded
        status_m = re.search(r'(\d+)\s*(day|hour|minute)s?\s*left\s*[•·]\s*(\d+)%\s*funded', block)
        if status_m:
            proj["days_left_text"] = f"{status_m.group(1)} {status_m.group(2)}s"
            proj["percent_funded"] = float(status_m.group(3))
        
        # Also try "hours left" pattern
        if "days_left_text" not in proj:
            status_m2 = re.search(r'(\d+)\s*(hour|minute)s?\s*left\s*[•·]\s*(\d+)%\s*funded', block)
            if status_m2:
                proj["days_left_text"] = f"{status_m2.group(1)} {status_m2.group(2)}s"
                proj["percent_funded"] = float(status_m2.group(3))
        
        # Pledged amount
        pledged_m = re.search(r'\$([\d,]+)\s*pledged', block)
        if pledged_m:
            proj["pledged_text"] = pledged_m.group(1).replace(",", "")
        
        # Goal
        goal_m = re.search(r'pledged of \$([\d,]+)\s*goal', block)
        if goal_m:
            proj["goal_text"] = goal_m.group(1).replace(",", "")
        
        # Backers
        backers_m = re.search(r'(\d+)\s*backers?', block)
        if backers_m:
            proj["backers"] = int(backers_m.group(1))
        
        # Location
        location_m = re.search(r'project-card__location[^>]*>(.*?)<', block, re.DOTALL)
        if location_m:
            proj["location"] = location_m.group(1).strip()
        
        if proj.get("name"):
            projects.append(proj)
    
    return projects

def json_project_to_entry(p):
    """Convert a JSON project to output entry."""
    return {
        "name": p.get("name"),
        "url": p.get("urls", {}).get("web", {}).get("project", ""),
        "pledged_usd": p.get("usd_pledged") or p.get("converted_pledged_amount"),
        "pledged_native": p.get("pledged"),
        "currency": p.get("currency"),
        "goal_native": p.get("goal"),
        "backers": p.get("backers_count"),
        "deadline_unix": p.get("deadline"),
        "percent_funded": p.get("percent_funded"),
        "state": p.get("state"),
        "blurb": p.get("blurb"),
        "country": p.get("country_displayable_name"),
        "creator": p.get("creator", {}).get("name") if p.get("creator") else None,
        "category": p.get("category", {}).get("name") if p.get("category") else None,
        "location": p.get("location", {}).get("name") if p.get("location") else None,
        "staff_pick": p.get("staff_pick"),
        "created_at": p.get("created_at"),
        "launched_at": p.get("launched_at"),
        "source": "json"
    }

def html_project_to_entry(p):
    """Convert an HTML-extracted project to output entry."""
    return {
        "name": p.get("name"),
        "url": p.get("url", ""),
        "pledged_usd": None,
        "pledged_native": None,
        "currency": "USD",
        "goal_native": None,
        "backers": p.get("backers"),
        "deadline_unix": None,
        "percent_funded": p.get("percent_funded"),
        "state": "live",
        "blurb": None,
        "country": None,
        "creator": p.get("creator"),
        "category": "Tabletop Games",
        "location": p.get("location"),
        "staff_pick": None,
        "created_at": None,
        "launched_at": None,
        "days_left_text": p.get("days_left_text"),
        "pledged_text": p.get("pledged_text"),
        "goal_text": p.get("goal_text"),
        "source": "html"
    }

def main():
    all_projects = []
    seen_urls = set()
    
    for page in range(1, 11):  # Try up to 10 pages
        print(f"Fetching page {page}...")
        try:
            html_content = fetch_page(page)
        except Exception as e:
            print(f"  ERROR on page {page}: {e}")
            continue
        
        has_json = 'data-project' in html_content
        print(f"  HTML: {len(html_content)} bytes, has data-project: {has_json}")
        
        if has_json:
            projects = extract_from_json(html_content)
            print(f"  Found {len(projects)} projects (JSON)")
            for p in projects:
                entry = json_project_to_entry(p)
                url = entry["url"]
                if url and url not in seen_urls:
                    seen_urls.add(url)
                    all_projects.append(entry)
        else:
            projects = extract_from_html(html_content)
            print(f"  Found {len(projects)} projects (HTML)")
            for p in projects:
                entry = html_project_to_entry(p)
                url = entry["url"]
                if url and url not in seen_urls:
                    seen_urls.add(url)
                    all_projects.append(entry)
        
        if page < 10:
            time.sleep(2)
    
    print(f"\n{'='*60}")
    print(f"Total unique projects collected: {len(all_projects)}")
    
    with open(OUTPUT_FILE, "w") as f:
        json.dump(all_projects, f, indent=2, ensure_ascii=False)
    
    print(f"Saved to {OUTPUT_FILE}")
    
    # Print summary
    for i, p in enumerate(all_projects[:10]):
        src = p.get("source", "?")
        print(f"\n{i+1}. [{src}] {p['name']}")
        print(f"   URL: {p['url']}")
        if p.get("pledged_usd"):
            print(f"   USD: ${p['pledged_usd']} / Goal: {p['goal_native']} {p['currency']}")
        if p.get("backers"):
            print(f"   Backers: {p['backers']}", end="")
        if p.get("percent_funded"):
            print(f" | Funded: {p['percent_funded']}%", end="")
        print()
    
    if len(all_projects) > 10:
        print(f"\n... and {len(all_projects)-10} more")

if __name__ == "__main__":
    main()