chore: vollständiger Workspace-Snapshot 03.07.2026
- publish_gatekeeper.py (neues Gatekeeper-System) - 4 Premium-Artikel: Veggie Match, Berlin 1960, Wild Tiled West, Scream Park - Alle kumulierten Artikel, Scripts, Medien und Caches
This commit is contained in:
Hermes Agent
+223
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch ALL Kickstarter board game campaigns via solver.
|
||||
Handles both data-project JSON and HTML-only project cards."""
|
||||
import json, re, html as html_mod, time, urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SOLVER_URL = "http://localhost:8191/v1"
|
||||
OUTPUT_FILE = "/home/hermes/workspace/ks_all_kw27_full.json"
|
||||
|
||||
def fetch_page(page_num):
|
||||
"""Fetch one page of KS discover via solver."""
|
||||
url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page_num}"
|
||||
payload = json.dumps({
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": 60000
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(SOLVER_URL, data=payload, headers={"Content-Type": "application/json"})
|
||||
resp = urllib.request.urlopen(req, timeout=90)
|
||||
data = json.loads(resp.read())
|
||||
|
||||
if data.get("status") != "ok":
|
||||
raise Exception(f"Solver error: {data}")
|
||||
|
||||
return data["solution"]["response"]
|
||||
|
||||
def extract_from_json(html_content):
|
||||
"""Extract projects from data-project JSON attributes."""
|
||||
data_proj = re.findall(r'data-project="({.*?})"', html_content)
|
||||
projects = []
|
||||
for dp in data_proj:
|
||||
decoded = html_mod.unescape(dp)
|
||||
try:
|
||||
proj = json.loads(decoded)
|
||||
projects.append(proj)
|
||||
except Exception:
|
||||
try:
|
||||
decoded2 = html_mod.unescape(decoded)
|
||||
proj = json.loads(decoded2)
|
||||
projects.append(proj)
|
||||
except Exception as e2:
|
||||
print(f" JSON parse error: {e2}")
|
||||
return projects
|
||||
|
||||
def extract_from_html(html_content):
|
||||
"""Extract projects from HTML project cards (fallback when no JSON)."""
|
||||
cards = re.findall(
|
||||
r'<div[^>]*project-card-root[^>]*>(.*?)</div>\s*</div>\s*</div>\s*</div>\s*</div>',
|
||||
html_content, re.DOTALL
|
||||
)
|
||||
|
||||
projects = []
|
||||
for block in cards:
|
||||
proj = {}
|
||||
|
||||
# Title
|
||||
title_m = re.search(r'project-card__title[^>]*>(.*?)</a>', block, re.DOTALL)
|
||||
if title_m:
|
||||
proj["name"] = html_mod.unescape(title_m.group(1).strip())
|
||||
|
||||
# URL
|
||||
url_m = re.search(r'href="(https?://www\.kickstarter\.com/projects/[^"?]+)', block)
|
||||
if url_m:
|
||||
proj["url"] = url_m.group(1)
|
||||
|
||||
# Creator
|
||||
creator_m = re.search(r'project-card__creator[^>]*>.*?<span[^>]*>(.*?)</span>', block, re.DOTALL)
|
||||
if creator_m:
|
||||
proj["creator"] = creator_m.group(1).strip()
|
||||
|
||||
# Days left / percent funded
|
||||
status_m = re.search(r'(\d+)\s*(day|hour|minute)s?\s*left\s*[•·]\s*(\d+)%\s*funded', block)
|
||||
if status_m:
|
||||
proj["days_left_text"] = f"{status_m.group(1)} {status_m.group(2)}s"
|
||||
proj["percent_funded"] = float(status_m.group(3))
|
||||
|
||||
# Also try "hours left" pattern
|
||||
if "days_left_text" not in proj:
|
||||
status_m2 = re.search(r'(\d+)\s*(hour|minute)s?\s*left\s*[•·]\s*(\d+)%\s*funded', block)
|
||||
if status_m2:
|
||||
proj["days_left_text"] = f"{status_m2.group(1)} {status_m2.group(2)}s"
|
||||
proj["percent_funded"] = float(status_m2.group(3))
|
||||
|
||||
# Pledged amount
|
||||
pledged_m = re.search(r'\$([\d,]+)\s*pledged', block)
|
||||
if pledged_m:
|
||||
proj["pledged_text"] = pledged_m.group(1).replace(",", "")
|
||||
|
||||
# Goal
|
||||
goal_m = re.search(r'pledged of \$([\d,]+)\s*goal', block)
|
||||
if goal_m:
|
||||
proj["goal_text"] = goal_m.group(1).replace(",", "")
|
||||
|
||||
# Backers
|
||||
backers_m = re.search(r'(\d+)\s*backers?', block)
|
||||
if backers_m:
|
||||
proj["backers"] = int(backers_m.group(1))
|
||||
|
||||
# Location
|
||||
location_m = re.search(r'project-card__location[^>]*>(.*?)<', block, re.DOTALL)
|
||||
if location_m:
|
||||
proj["location"] = location_m.group(1).strip()
|
||||
|
||||
if proj.get("name"):
|
||||
projects.append(proj)
|
||||
|
||||
return projects
|
||||
|
||||
def json_project_to_entry(p):
|
||||
"""Convert a JSON project to output entry."""
|
||||
return {
|
||||
"name": p.get("name"),
|
||||
"url": p.get("urls", {}).get("web", {}).get("project", ""),
|
||||
"pledged_usd": p.get("usd_pledged") or p.get("converted_pledged_amount"),
|
||||
"pledged_native": p.get("pledged"),
|
||||
"currency": p.get("currency"),
|
||||
"goal_native": p.get("goal"),
|
||||
"backers": p.get("backers_count"),
|
||||
"deadline_unix": p.get("deadline"),
|
||||
"percent_funded": p.get("percent_funded"),
|
||||
"state": p.get("state"),
|
||||
"blurb": p.get("blurb"),
|
||||
"country": p.get("country_displayable_name"),
|
||||
"creator": p.get("creator", {}).get("name") if p.get("creator") else None,
|
||||
"category": p.get("category", {}).get("name") if p.get("category") else None,
|
||||
"location": p.get("location", {}).get("name") if p.get("location") else None,
|
||||
"staff_pick": p.get("staff_pick"),
|
||||
"created_at": p.get("created_at"),
|
||||
"launched_at": p.get("launched_at"),
|
||||
"source": "json"
|
||||
}
|
||||
|
||||
def html_project_to_entry(p):
|
||||
"""Convert an HTML-extracted project to output entry."""
|
||||
return {
|
||||
"name": p.get("name"),
|
||||
"url": p.get("url", ""),
|
||||
"pledged_usd": None,
|
||||
"pledged_native": None,
|
||||
"currency": "USD",
|
||||
"goal_native": None,
|
||||
"backers": p.get("backers"),
|
||||
"deadline_unix": None,
|
||||
"percent_funded": p.get("percent_funded"),
|
||||
"state": "live",
|
||||
"blurb": None,
|
||||
"country": None,
|
||||
"creator": p.get("creator"),
|
||||
"category": "Tabletop Games",
|
||||
"location": p.get("location"),
|
||||
"staff_pick": None,
|
||||
"created_at": None,
|
||||
"launched_at": None,
|
||||
"days_left_text": p.get("days_left_text"),
|
||||
"pledged_text": p.get("pledged_text"),
|
||||
"goal_text": p.get("goal_text"),
|
||||
"source": "html"
|
||||
}
|
||||
|
||||
def main():
|
||||
all_projects = []
|
||||
seen_urls = set()
|
||||
|
||||
for page in range(1, 11): # Try up to 10 pages
|
||||
print(f"Fetching page {page}...")
|
||||
try:
|
||||
html_content = fetch_page(page)
|
||||
except Exception as e:
|
||||
print(f" ERROR on page {page}: {e}")
|
||||
continue
|
||||
|
||||
has_json = 'data-project' in html_content
|
||||
print(f" HTML: {len(html_content)} bytes, has data-project: {has_json}")
|
||||
|
||||
if has_json:
|
||||
projects = extract_from_json(html_content)
|
||||
print(f" Found {len(projects)} projects (JSON)")
|
||||
for p in projects:
|
||||
entry = json_project_to_entry(p)
|
||||
url = entry["url"]
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
all_projects.append(entry)
|
||||
else:
|
||||
projects = extract_from_html(html_content)
|
||||
print(f" Found {len(projects)} projects (HTML)")
|
||||
for p in projects:
|
||||
entry = html_project_to_entry(p)
|
||||
url = entry["url"]
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
all_projects.append(entry)
|
||||
|
||||
if page < 10:
|
||||
time.sleep(2)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Total unique projects collected: {len(all_projects)}")
|
||||
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
json.dump(all_projects, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"Saved to {OUTPUT_FILE}")
|
||||
|
||||
# Print summary
|
||||
for i, p in enumerate(all_projects[:10]):
|
||||
src = p.get("source", "?")
|
||||
print(f"\n{i+1}. [{src}] {p['name']}")
|
||||
print(f" URL: {p['url']}")
|
||||
if p.get("pledged_usd"):
|
||||
print(f" USD: ${p['pledged_usd']} / Goal: {p['goal_native']} {p['currency']}")
|
||||
if p.get("backers"):
|
||||
print(f" Backers: {p['backers']}", end="")
|
||||
if p.get("percent_funded"):
|
||||
print(f" | Funded: {p['percent_funded']}%", end="")
|
||||
print()
|
||||
|
||||
if len(all_projects) > 10:
|
||||
print(f"\n... and {len(all_projects)-10} more")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user