fix: <style>-Blöcke aus allen 24 Artikeln entfernt + Skill korrigiert
- artikel_*.html: 24 Dateien von <style>-Blöcken befreit (body, max-width zerstören Joomla) - journalistic-article-writing SKILL.md: falsche 'AUSNAHME 09.07.' entfernt - Ursache: Skill-Kommentar erlaubte Sub-Agenten <style> 'für Vorschau' — kein Gatekeeper existiert
This commit is contained in:
Hermes Agent
+63
-214
@@ -1,223 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch ALL Kickstarter board game campaigns via solver.
|
||||
Handles both data-project JSON and HTML-only project cards."""
|
||||
import json, re, html as html_mod, time, urllib.request
|
||||
from datetime import datetime, timezone
|
||||
"""Fetch Kickstarter data via FlareSolverr — handles HTML-wrapped JSON"""
|
||||
import urllib.request, json, time, sys, re
|
||||
|
||||
SOLVER_URL = "http://localhost:8191/v1"
|
||||
OUTPUT_FILE = "/home/hermes/workspace/ks_all_kw27_full.json"
|
||||
FLARE = "http://localhost:8191/v1"
|
||||
all_projects = []
|
||||
|
||||
def fetch_page(page_num):
|
||||
"""Fetch one page of KS discover via solver."""
|
||||
url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page_num}"
|
||||
payload = json.dumps({
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": 60000
|
||||
}).encode()
|
||||
for page in range(1, 6):
|
||||
url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&page={page}&format=json"
|
||||
payload = json.dumps({"cmd": "request.get", "url": url, "maxTimeout": 60000}).encode()
|
||||
|
||||
req = urllib.request.Request(SOLVER_URL, data=payload, headers={"Content-Type": "application/json"})
|
||||
resp = urllib.request.urlopen(req, timeout=90)
|
||||
data = json.loads(resp.read())
|
||||
req = urllib.request.Request(FLARE, data=payload, headers={"Content-Type": "application/json"}, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=90) as resp:
|
||||
fs_resp = json.loads(resp.read())
|
||||
except Exception as e:
|
||||
print(f"Page {page}: ERROR {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
if data.get("status") != "ok":
|
||||
raise Exception(f"Solver error: {data}")
|
||||
if fs_resp.get('status') != 'ok':
|
||||
print(f"Page {page}: FS status={fs_resp.get('status')}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
return data["solution"]["response"]
|
||||
raw = fs_resp['solution']['response']
|
||||
|
||||
# Extract JSON from <pre> tag if wrapped in HTML
|
||||
if '<pre>' in raw:
|
||||
match = re.search(r'<pre>\s*(.+?)\s*</pre>', raw, re.DOTALL)
|
||||
if match:
|
||||
raw = match.group(1)
|
||||
|
||||
try:
|
||||
inner = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Page {page}: JSON decode error: {e}, first 200 chars: {raw[:200]}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
projects = inner.get('projects', [])
|
||||
print(f"Page {page}: {len(projects)} projects")
|
||||
|
||||
for p in projects:
|
||||
photo = ''
|
||||
if isinstance(p.get('photo'), dict):
|
||||
photo = p['photo'].get('1024x576', '')
|
||||
|
||||
all_projects.append({
|
||||
'name': p.get('name', ''),
|
||||
'slug': p.get('slug', ''),
|
||||
'url': f"https://www.kickstarter.com/projects/{p.get('slug','')}",
|
||||
'photo': photo,
|
||||
'pledged': float(p.get('pledged', 0) or 0),
|
||||
'goal': float(p.get('goal', 0) or 0),
|
||||
'backers': int(p.get('backers_count', 0) or 0),
|
||||
'deadline': p.get('deadline', ''),
|
||||
'state': p.get('state', ''),
|
||||
'currency': p.get('currency', 'USD'),
|
||||
})
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
def extract_from_json(html_content):
|
||||
"""Extract projects from data-project JSON attributes."""
|
||||
data_proj = re.findall(r'data-project="({.*?})"', html_content)
|
||||
projects = []
|
||||
for dp in data_proj:
|
||||
decoded = html_mod.unescape(dp)
|
||||
try:
|
||||
proj = json.loads(decoded)
|
||||
projects.append(proj)
|
||||
except Exception:
|
||||
try:
|
||||
decoded2 = html_mod.unescape(decoded)
|
||||
proj = json.loads(decoded2)
|
||||
projects.append(proj)
|
||||
except Exception as e2:
|
||||
print(f" JSON parse error: {e2}")
|
||||
return projects
|
||||
# Filter live
|
||||
live = [p for p in all_projects if p['state'] == 'live']
|
||||
print(f"\nTotal: {len(all_projects)}, Live: {len(live)}")
|
||||
|
||||
def extract_from_html(html_content):
|
||||
"""Extract projects from HTML project cards (fallback when no JSON)."""
|
||||
cards = re.findall(
|
||||
r'<div[^>]*project-card-root[^>]*>(.*?)</div>\s*</div>\s*</div>\s*</div>\s*</div>',
|
||||
html_content, re.DOTALL
|
||||
)
|
||||
|
||||
projects = []
|
||||
for block in cards:
|
||||
proj = {}
|
||||
|
||||
# Title
|
||||
title_m = re.search(r'project-card__title[^>]*>(.*?)</a>', block, re.DOTALL)
|
||||
if title_m:
|
||||
proj["name"] = html_mod.unescape(title_m.group(1).strip())
|
||||
|
||||
# URL
|
||||
url_m = re.search(r'href="(https?://www\.kickstarter\.com/projects/[^"?]+)', block)
|
||||
if url_m:
|
||||
proj["url"] = url_m.group(1)
|
||||
|
||||
# Creator
|
||||
creator_m = re.search(r'project-card__creator[^>]*>.*?<span[^>]*>(.*?)</span>', block, re.DOTALL)
|
||||
if creator_m:
|
||||
proj["creator"] = creator_m.group(1).strip()
|
||||
|
||||
# Days left / percent funded
|
||||
status_m = re.search(r'(\d+)\s*(day|hour|minute)s?\s*left\s*[•·]\s*(\d+)%\s*funded', block)
|
||||
if status_m:
|
||||
proj["days_left_text"] = f"{status_m.group(1)} {status_m.group(2)}s"
|
||||
proj["percent_funded"] = float(status_m.group(3))
|
||||
|
||||
# Also try "hours left" pattern
|
||||
if "days_left_text" not in proj:
|
||||
status_m2 = re.search(r'(\d+)\s*(hour|minute)s?\s*left\s*[•·]\s*(\d+)%\s*funded', block)
|
||||
if status_m2:
|
||||
proj["days_left_text"] = f"{status_m2.group(1)} {status_m2.group(2)}s"
|
||||
proj["percent_funded"] = float(status_m2.group(3))
|
||||
|
||||
# Pledged amount
|
||||
pledged_m = re.search(r'\$([\d,]+)\s*pledged', block)
|
||||
if pledged_m:
|
||||
proj["pledged_text"] = pledged_m.group(1).replace(",", "")
|
||||
|
||||
# Goal
|
||||
goal_m = re.search(r'pledged of \$([\d,]+)\s*goal', block)
|
||||
if goal_m:
|
||||
proj["goal_text"] = goal_m.group(1).replace(",", "")
|
||||
|
||||
# Backers
|
||||
backers_m = re.search(r'(\d+)\s*backers?', block)
|
||||
if backers_m:
|
||||
proj["backers"] = int(backers_m.group(1))
|
||||
|
||||
# Location
|
||||
location_m = re.search(r'project-card__location[^>]*>(.*?)<', block, re.DOTALL)
|
||||
if location_m:
|
||||
proj["location"] = location_m.group(1).strip()
|
||||
|
||||
if proj.get("name"):
|
||||
projects.append(proj)
|
||||
|
||||
return projects
|
||||
# Save
|
||||
with open('/home/hermes/workspace/ks_data.json', 'w') as f:
|
||||
json.dump(live, f, indent=2, ensure_ascii=False)
|
||||
|
||||
def json_project_to_entry(p):
|
||||
"""Convert a JSON project to output entry."""
|
||||
return {
|
||||
"name": p.get("name"),
|
||||
"url": p.get("urls", {}).get("web", {}).get("project", ""),
|
||||
"pledged_usd": p.get("usd_pledged") or p.get("converted_pledged_amount"),
|
||||
"pledged_native": p.get("pledged"),
|
||||
"currency": p.get("currency"),
|
||||
"goal_native": p.get("goal"),
|
||||
"backers": p.get("backers_count"),
|
||||
"deadline_unix": p.get("deadline"),
|
||||
"percent_funded": p.get("percent_funded"),
|
||||
"state": p.get("state"),
|
||||
"blurb": p.get("blurb"),
|
||||
"country": p.get("country_displayable_name"),
|
||||
"creator": p.get("creator", {}).get("name") if p.get("creator") else None,
|
||||
"category": p.get("category", {}).get("name") if p.get("category") else None,
|
||||
"location": p.get("location", {}).get("name") if p.get("location") else None,
|
||||
"staff_pick": p.get("staff_pick"),
|
||||
"created_at": p.get("created_at"),
|
||||
"launched_at": p.get("launched_at"),
|
||||
"source": "json"
|
||||
}
|
||||
|
||||
def html_project_to_entry(p):
|
||||
"""Convert an HTML-extracted project to output entry."""
|
||||
return {
|
||||
"name": p.get("name"),
|
||||
"url": p.get("url", ""),
|
||||
"pledged_usd": None,
|
||||
"pledged_native": None,
|
||||
"currency": "USD",
|
||||
"goal_native": None,
|
||||
"backers": p.get("backers"),
|
||||
"deadline_unix": None,
|
||||
"percent_funded": p.get("percent_funded"),
|
||||
"state": "live",
|
||||
"blurb": None,
|
||||
"country": None,
|
||||
"creator": p.get("creator"),
|
||||
"category": "Tabletop Games",
|
||||
"location": p.get("location"),
|
||||
"staff_pick": None,
|
||||
"created_at": None,
|
||||
"launched_at": None,
|
||||
"days_left_text": p.get("days_left_text"),
|
||||
"pledged_text": p.get("pledged_text"),
|
||||
"goal_text": p.get("goal_text"),
|
||||
"source": "html"
|
||||
}
|
||||
|
||||
def main():
|
||||
all_projects = []
|
||||
seen_urls = set()
|
||||
|
||||
for page in range(1, 11): # Try up to 10 pages
|
||||
print(f"Fetching page {page}...")
|
||||
try:
|
||||
html_content = fetch_page(page)
|
||||
except Exception as e:
|
||||
print(f" ERROR on page {page}: {e}")
|
||||
continue
|
||||
|
||||
has_json = 'data-project' in html_content
|
||||
print(f" HTML: {len(html_content)} bytes, has data-project: {has_json}")
|
||||
|
||||
if has_json:
|
||||
projects = extract_from_json(html_content)
|
||||
print(f" Found {len(projects)} projects (JSON)")
|
||||
for p in projects:
|
||||
entry = json_project_to_entry(p)
|
||||
url = entry["url"]
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
all_projects.append(entry)
|
||||
else:
|
||||
projects = extract_from_html(html_content)
|
||||
print(f" Found {len(projects)} projects (HTML)")
|
||||
for p in projects:
|
||||
entry = html_project_to_entry(p)
|
||||
url = entry["url"]
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
all_projects.append(entry)
|
||||
|
||||
if page < 10:
|
||||
time.sleep(2)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Total unique projects collected: {len(all_projects)}")
|
||||
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
json.dump(all_projects, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"Saved to {OUTPUT_FILE}")
|
||||
|
||||
# Print summary
|
||||
for i, p in enumerate(all_projects[:10]):
|
||||
src = p.get("source", "?")
|
||||
print(f"\n{i+1}. [{src}] {p['name']}")
|
||||
print(f" URL: {p['url']}")
|
||||
if p.get("pledged_usd"):
|
||||
print(f" USD: ${p['pledged_usd']} / Goal: {p['goal_native']} {p['currency']}")
|
||||
if p.get("backers"):
|
||||
print(f" Backers: {p['backers']}", end="")
|
||||
if p.get("percent_funded"):
|
||||
print(f" | Funded: {p['percent_funded']}%", end="")
|
||||
print()
|
||||
|
||||
if len(all_projects) > 10:
|
||||
print(f"\n... and {len(all_projects)-10} more")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
# Top 5
|
||||
for i, p in enumerate(sorted(live, key=lambda x: x['pledged'], reverse=True)[:5]):
|
||||
pct = round(p['pledged']/max(p['goal'],1)*100, 1)
|
||||
print(f"{i+1}. {p['name']}: ${p['pledged']:,.0f} ({pct}%) - {p['backers']} backers")
|
||||
|
||||
Reference in New Issue
Block a user