#!/usr/bin/env python3
"""Fetch Kickstarter data via FlareSolverr — handles HTML-wrapped JSON"""
import urllib.request, json, time, sys, re

FLARE = "http://localhost:8191/v1"
all_projects = []

for page in range(1, 6):
    url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page}&format=json"
    payload = json.dumps({"cmd": "request.get", "url": url, "maxTimeout": 60000}).encode()
    
    req = urllib.request.Request(FLARE, data=payload, headers={"Content-Type": "application/json"}, method="POST")
    try:
        with urllib.request.urlopen(req, timeout=90) as resp:
            fs_resp = json.loads(resp.read())
    except Exception as e:
        print(f"Page {page}: ERROR {e}", file=sys.stderr)
        continue
    
    if fs_resp.get('status') != 'ok':
        print(f"Page {page}: FS status={fs_resp.get('status')}", file=sys.stderr)
        continue
    
    raw = fs_resp['solution']['response']
    
    # Extract JSON from <pre> tag if wrapped in HTML
    if '<pre>' in raw:
        match = re.search(r'<pre>\s*(.+?)\s*</pre>', raw, re.DOTALL)
        if match:
            raw = match.group(1)
    
    try:
        inner = json.loads(raw)
    except json.JSONDecodeError as e:
        print(f"Page {page}: JSON decode error: {e}, first 200 chars: {raw[:200]}", file=sys.stderr)
        continue
    
    projects = inner.get('projects', [])
    print(f"Page {page}: {len(projects)} projects")
    
    for p in projects:
        photo = ''
        if isinstance(p.get('photo'), dict):
            photo = p['photo'].get('1024x576', '')
        
        all_projects.append({
            'name': p.get('name', ''),
            'slug': p.get('slug', ''),
            'url': f"https://www.kickstarter.com/projects/{p.get('slug','')}",
            'photo': photo,
            'pledged': float(p.get('pledged', 0) or 0),
            'goal': float(p.get('goal', 0) or 0),
            'backers': int(p.get('backers_count', 0) or 0),
            'deadline': p.get('deadline', ''),
            'state': p.get('state', ''),
            'currency': p.get('currency', 'USD'),
        })
    
    time.sleep(2)

# Filter live
live = [p for p in all_projects if p['state'] == 'live']
print(f"\nTotal: {len(all_projects)}, Live: {len(live)}")

# Save
with open('/home/hermes/workspace/ks_data.json', 'w') as f:
    json.dump(live, f, indent=2, ensure_ascii=False)

# Top 5
for i, p in enumerate(sorted(live, key=lambda x: x['pledged'], reverse=True)[:5]):
    pct = round(p['pledged']/max(p['goal'],1)*100, 1)
    print(f"{i+1}. {p['name']}: ${p['pledged']:,.0f} ({pct}%) - {p['backers']} backers")