feat: Review-Builder — Word-Datei + Bilder hochladen, LLM-strukturiert HTML bauen
- GET /review-builder: Upload-Formular (Word .docx + Bilder Drag/Drop + Metadaten)
- POST /review-builder/build: Word parsen → LLM (DeepSeek Flash) strukturiert
Teaser/Spielerklärung/Fazit → HTML bauen → im Workspace speichern
- Bild-Resize: 1920x1080 @ 72dpi (PIL), Bilderverzeichnis/01/{buchstabe}/{spiel}/
- review_config.json: 25+ Autoren, 5 Disclosures, 5 Rubriken, Komplexitätsstufen
- Live-Publish via Joomla-API optional (Button: Vorschau / Veröffentlichen)
- Link im Dashboard: 📝 Review-Builder
This commit is contained in:
Hermes Agent
+100
-59
@@ -1,72 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch Kickstarter data via FlareSolverr — handles HTML-wrapped JSON"""
|
||||
import urllib.request, json, time, sys, re
|
||||
"""Fetch Kickstarter Discover API pages 1-5 via FlareSolverr, extract & filter projects."""
|
||||
import urllib.request, json, sys, os, re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
FLARESOLVERR = "http://localhost:8191/v1"
|
||||
SKIP_KW = ['5e', '5th edition', 'handbook', 'rpg', 'd&d', 'dnd', 'roleplaying game', 'ttrpg',
|
||||
'dungeon master', 'stl', '15mm', '28mm', '32mm', 'resin', 'miniature',
|
||||
'plastic regiment', 'multi-part', '3d print', 'terrain',
|
||||
'dice bag', 'dice tray', 'playmat', 'map pack', 'battle map',
|
||||
'touchscreen', 'mapcase', 'character sheet', 'spellbook',
|
||||
'soundtrack', 'soundscapes', 'music album', 'props', 'makeup',
|
||||
'accessory']
|
||||
|
||||
def get_ks(url, session=None):
|
||||
payload = {"cmd": "request.get", "url": url, "maxTimeout": 45000}
|
||||
if session:
|
||||
payload["session"] = session
|
||||
req = urllib.request.Request(FLARESOLVERR,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
FLARE = "http://localhost:8191/v1"
|
||||
all_projects = []
|
||||
session = None
|
||||
|
||||
for page in range(1, 6):
|
||||
url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page}&format=json"
|
||||
payload = json.dumps({"cmd": "request.get", "url": url, "maxTimeout": 60000}).encode()
|
||||
|
||||
req = urllib.request.Request(FLARE, data=payload, headers={"Content-Type": "application/json"}, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=90) as resp:
|
||||
fs_resp = json.loads(resp.read())
|
||||
except Exception as e:
|
||||
print(f"Page {page}: ERROR {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
if fs_resp.get('status') != 'ok':
|
||||
print(f"Page {page}: FS status={fs_resp.get('status')}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
raw = fs_resp['solution']['response']
|
||||
|
||||
# Extract JSON from <pre> tag if wrapped in HTML
|
||||
if '<pre>' in raw:
|
||||
match = re.search(r'<pre>\s*(.+?)\s*</pre>', raw, re.DOTALL)
|
||||
if match:
|
||||
raw = match.group(1)
|
||||
print(f"Fetching KS page {page}...", flush=True)
|
||||
|
||||
try:
|
||||
inner = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Page {page}: JSON decode error: {e}, first 200 chars: {raw[:200]}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
projects = inner.get('projects', [])
|
||||
print(f"Page {page}: {len(projects)} projects")
|
||||
|
||||
for p in projects:
|
||||
photo = ''
|
||||
if isinstance(p.get('photo'), dict):
|
||||
photo = p['photo'].get('1024x576', '')
|
||||
result = get_ks(url, session=session)
|
||||
if result.get("status") != "ok":
|
||||
print(f" ERROR: {result.get('message', 'Unknown error')}", flush=True)
|
||||
continue
|
||||
|
||||
all_projects.append({
|
||||
'name': p.get('name', ''),
|
||||
'slug': p.get('slug', ''),
|
||||
'url': f"https://www.kickstarter.com/projects/{p.get('slug','')}",
|
||||
'photo': photo,
|
||||
'pledged': float(p.get('pledged', 0) or 0),
|
||||
'goal': float(p.get('goal', 0) or 0),
|
||||
'backers': int(p.get('backers_count', 0) or 0),
|
||||
'deadline': p.get('deadline', ''),
|
||||
'state': p.get('state', ''),
|
||||
'currency': p.get('currency', 'USD'),
|
||||
})
|
||||
# Save session for subsequent requests
|
||||
session = result.get("solution", {}).get("session")
|
||||
|
||||
resp_text = result["solution"]["response"]
|
||||
|
||||
# Extract JSON from HTML-wrapped response
|
||||
if "<pre>" in resp_text:
|
||||
match = re.search(r'<pre>\s*(.+?)\s*</pre>', resp_text, re.DOTALL)
|
||||
if match:
|
||||
raw_json = match.group(1)
|
||||
else:
|
||||
print(f" Could not extract JSON from <pre>", flush=True)
|
||||
continue
|
||||
else:
|
||||
raw_json = resp_text
|
||||
|
||||
inner = json.loads(raw_json)
|
||||
projects = inner.get("projects", [])
|
||||
|
||||
for p in projects:
|
||||
name = (p.get("name") or "").lower()
|
||||
blurb = (p.get("blurb") or "").lower()
|
||||
combined = name + " " + blurb
|
||||
is_skip = any(kw in combined for kw in SKIP_KW)
|
||||
if is_skip:
|
||||
continue
|
||||
|
||||
# Extract fields
|
||||
slug = p.get("slug", "")
|
||||
project_id = p.get("id")
|
||||
|
||||
# Deadline handling
|
||||
deadline_raw = p.get("deadline")
|
||||
if isinstance(deadline_raw, int):
|
||||
deadline_dt = datetime.fromtimestamp(deadline_raw, tz=timezone.utc)
|
||||
elif isinstance(deadline_raw, str):
|
||||
try:
|
||||
deadline_dt = datetime.fromisoformat(deadline_raw.replace('Z', '+00:00'))
|
||||
except:
|
||||
deadline_dt = None
|
||||
else:
|
||||
deadline_dt = None
|
||||
|
||||
# State check
|
||||
state = p.get("state", "live")
|
||||
|
||||
all_projects.append({
|
||||
"id": project_id,
|
||||
"name": p.get("name", ""),
|
||||
"slug": slug,
|
||||
"url": f"https://www.kickstarter.com/projects/{slug}" if slug else "",
|
||||
"blurb": p.get("blurb", ""),
|
||||
"pledged": p.get("pledged", 0),
|
||||
"goal": p.get("goal", 0),
|
||||
"backers_count": p.get("backers_count", 0),
|
||||
"currency": p.get("currency", "USD"),
|
||||
"deadline": deadline_dt.isoformat() if deadline_dt else None,
|
||||
"deadline_ts": p.get("deadline"),
|
||||
"photo": (p.get("photo") or {}).get("1024x768", ""),
|
||||
"state": state,
|
||||
"creator": (p.get("creator") or {}).get("name", ""),
|
||||
"location": (p.get("location") or {}).get("short", ""),
|
||||
})
|
||||
|
||||
print(f" Found {len(projects)} raw → {sum(1 for p in projects if not any(kw in ((p.get('name') or '') + ' ' + (p.get('blurb') or '')).lower() for kw in SKIP_KW))} filtered (page total: {len(all_projects)})", flush=True)
|
||||
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
print(f" EXCEPTION: {e}", flush=True)
|
||||
continue
|
||||
|
||||
# Filter live
|
||||
live = [p for p in all_projects if p['state'] == 'live']
|
||||
print(f"\nTotal: {len(all_projects)}, Live: {len(live)}")
|
||||
print(f"\nTotal KS board games: {len(all_projects)}", flush=True)
|
||||
|
||||
# Save
|
||||
with open('/home/hermes/workspace/ks_data.json', 'w') as f:
|
||||
json.dump(live, f, indent=2, ensure_ascii=False)
|
||||
with open("/home/hermes/workspace/ks_filtered.json", "w") as f:
|
||||
json.dump(all_projects, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Top 5
|
||||
for i, p in enumerate(sorted(live, key=lambda x: x['pledged'], reverse=True)[:5]):
|
||||
pct = round(p['pledged']/max(p['goal'],1)*100, 1)
|
||||
print(f"{i+1}. {p['name']}: ${p['pledged']:,.0f} ({pct}%) - {p['backers']} backers")
|
||||
print("SAVED to ks_filtered.json", flush=True)
|
||||
|
||||
Reference in New Issue
Block a user