chore: vollständiger Workspace-Snapshot 03.07.2026
- publish_gatekeeper.py (neues Gatekeeper-System) - 4 Premium-Artikel: Veggie Match, Berlin 1960, Wild Tiled West, Scream Park - Alle kumulierten Artikel, Scripts, Medien und Caches
This commit is contained in:
Hermes Agent
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Collect ALL active Kickstarter board game campaigns (category_id=34).
|
||||
Uses Discover API via FlareSolverr + fetch_campaign() for top-20 live data.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import requests
|
||||
|
||||
# Add workspace to path for kickstarter_fetcher import
|
||||
sys.path.insert(0, '/home/hermes/workspace/bsn-chatbot')
|
||||
from kickstarter_fetcher import fetch_campaign, status_report
|
||||
|
||||
FLARESOLVERR_URL = "http://localhost:8191/v1"
|
||||
OUTPUT_FILE = "/home/hermes/workspace/ks_all_kw27.json"
|
||||
|
||||
# Keywords to filter OUT (non-board-game campaigns)
|
||||
EXCLUDE_KEYWORDS = [
|
||||
"stl", "3d print", "3d printable", "miniature", "miniatures",
|
||||
"rpg book", "rpg supplement", "rpg zine", "rpg tracker",
|
||||
"dice", "dice set", "dice tower",
|
||||
"playing cards", "tarot", "poker deck",
|
||||
"ttrpg", "5e", "d&d", "dungeons & dragons", "pathfinder",
|
||||
"terrain", "scenery",
|
||||
]
|
||||
|
||||
# Keywords that suggest it IS a board game
|
||||
BOARD_GAME_KEYWORDS = [
|
||||
"board game", "card game", "deck-building", "deck building",
|
||||
"worker placement", "euro", "strategy game", "family game",
|
||||
"cooperative game", "co-op game", "tile", "dice game",
|
||||
"wargame", "war game", "miniatures game", "skirmish",
|
||||
"tabletop game", "party game", "roll & write", "roll and write",
|
||||
"drafting", "area control", "engine builder", "engine building",
|
||||
"legacy", "campaign game", "adventure game",
|
||||
]
|
||||
|
||||
def fetch_discover_page(page_num):
|
||||
"""Fetch one page of the Kickstarter Discover API via FlareSolverr."""
|
||||
url = f"https://www.kickstarter.com/discover/advanced?category_id=34&sort=end_date&state=live&format=json&page={page_num}"
|
||||
|
||||
try:
|
||||
r = requests.post(
|
||||
FLARESOLVERR_URL,
|
||||
json={
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": 60000,
|
||||
},
|
||||
timeout=90,
|
||||
)
|
||||
data = r.json()
|
||||
if data.get("status") != "ok":
|
||||
print(f" Page {page_num}: FlareSolverr error: {data.get('message', '?')}")
|
||||
return []
|
||||
|
||||
html = data["solution"]["response"]
|
||||
m = re.search(r'<pre>(.*?)</pre>', html, re.DOTALL)
|
||||
if not m:
|
||||
print(f" Page {page_num}: No JSON found in response")
|
||||
return []
|
||||
|
||||
d = json.loads(m.group(1))
|
||||
projects = d.get("projects", [])
|
||||
print(f" Page {page_num}: {len(projects)} projects (total hits: {d.get('total_hits')})")
|
||||
return projects
|
||||
except Exception as e:
|
||||
print(f" Page {page_num}: Exception: {e}")
|
||||
return []
|
||||
|
||||
def is_board_game(project):
|
||||
"""Heuristic filter: is this actually a board game?"""
|
||||
name = (project.get("name") or "").lower()
|
||||
blurb = (project.get("blurb") or "").lower()
|
||||
combined = name + " " + blurb
|
||||
|
||||
# Check exclusions first
|
||||
for kw in EXCLUDE_KEYWORDS:
|
||||
if kw in combined:
|
||||
# But if it also has strong board game keywords, keep it
|
||||
for bg_kw in BOARD_GAME_KEYWORDS:
|
||||
if bg_kw in combined:
|
||||
break
|
||||
else:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def parse_project(project):
|
||||
"""Convert Discover API project to our format. Converts to USD."""
|
||||
urls = project.get("urls", {}).get("web", {})
|
||||
proj_url = urls.get("project", "")
|
||||
|
||||
# Parse deadline
|
||||
deadline_ts = project.get("deadline", 0)
|
||||
deadline_str = ""
|
||||
days_left = -1
|
||||
if deadline_ts:
|
||||
from datetime import datetime, timezone
|
||||
dt = datetime.fromtimestamp(deadline_ts, tz=timezone.utc)
|
||||
deadline_str = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
delta = dt - datetime.now(timezone.utc)
|
||||
days_left = max(0, delta.days)
|
||||
|
||||
# Convert to USD: Discover API returns amounts in local currency
|
||||
# static_usd_rate converts local currency → USD
|
||||
local_pledged = project.get("pledged", 0) or 0
|
||||
local_goal = project.get("goal", 0) or 0
|
||||
usd_rate = project.get("static_usd_rate", 1.0) or 1.0
|
||||
|
||||
pledged_usd = round(local_pledged * usd_rate, 2)
|
||||
goal_usd = round(local_goal * usd_rate, 2)
|
||||
pct = round(pledged_usd / goal_usd * 100, 1) if goal_usd > 0 else 0
|
||||
|
||||
return {
|
||||
"name": project.get("name", ""),
|
||||
"url": proj_url,
|
||||
"pledged_usd": pledged_usd,
|
||||
"pledged_display": f"${pledged_usd:,.0f}",
|
||||
"goal_usd": goal_usd,
|
||||
"goal_display": f"${goal_usd:,.0f}",
|
||||
"percent_funded": pct,
|
||||
"backers": project.get("backers_count", 0),
|
||||
"days_left": days_left,
|
||||
"deadline": deadline_str,
|
||||
"blurb": project.get("blurb", ""),
|
||||
"country": project.get("country", ""),
|
||||
"currency": project.get("currency", ""),
|
||||
"state": project.get("state", "live"),
|
||||
"source": "discover_api",
|
||||
}
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("KS ALL Campaign Collector - KW27")
|
||||
print("=" * 60)
|
||||
|
||||
# Step 1: Fetch all pages
|
||||
print("\n[1] Fetching all Discover API pages...")
|
||||
all_projects = []
|
||||
page = 1
|
||||
while True:
|
||||
projects = fetch_discover_page(page)
|
||||
if not projects:
|
||||
break
|
||||
all_projects.extend(projects)
|
||||
page += 1
|
||||
time.sleep(1) # Be gentle
|
||||
|
||||
print(f"\nTotal raw projects fetched: {len(all_projects)}")
|
||||
|
||||
# Step 2: Filter for board games
|
||||
print("\n[2] Filtering for board games...")
|
||||
board_games = []
|
||||
excluded = []
|
||||
for p in all_projects:
|
||||
if is_board_game(p):
|
||||
board_games.append(p)
|
||||
else:
|
||||
excluded.append(p)
|
||||
|
||||
print(f" Board games: {len(board_games)}")
|
||||
print(f" Excluded: {len(excluded)}")
|
||||
if excluded:
|
||||
print(f" Excluded examples: {[e.get('name','')[:60] for e in excluded[:10]]}")
|
||||
|
||||
# Step 3: Parse all board games
|
||||
print("\n[3] Parsing board game data...")
|
||||
parsed = [parse_project(p) for p in board_games]
|
||||
|
||||
# Sort by pledged amount descending
|
||||
parsed.sort(key=lambda x: x["pledged_usd"], reverse=True)
|
||||
|
||||
# Step 4: Fetch top-20 with FlareSolverr for LIVE data
|
||||
print("\n[4] Fetching top-20 campaigns with FlareSolverr (LIVE data)...")
|
||||
top_n = min(20, len(parsed))
|
||||
for i in range(top_n):
|
||||
p = parsed[i]
|
||||
url = p["url"]
|
||||
print(f" [{i+1}/{top_n}] {p['name'][:60]}...")
|
||||
campaign = fetch_campaign(url, timeout=90)
|
||||
if campaign and campaign.source == "flaresolverr":
|
||||
# Update with live data
|
||||
p["pledged_display"] = campaign.pledged
|
||||
p["goal_display"] = campaign.goal
|
||||
p["backers"] = campaign.backers
|
||||
p["days_left"] = campaign.days_left
|
||||
p["status"] = campaign.status
|
||||
p["source"] = "flaresolverr_live"
|
||||
p["description"] = campaign.description
|
||||
# Also update numeric USD values from display strings
|
||||
try:
|
||||
p["pledged_usd"] = float(campaign.pledged.replace("$","").replace(",",""))
|
||||
except: pass
|
||||
try:
|
||||
p["goal_usd"] = float(campaign.goal.replace("$","").replace(",",""))
|
||||
except: pass
|
||||
# Recalculate percent
|
||||
if p["goal_usd"] > 0:
|
||||
p["percent_funded"] = round(p["pledged_usd"] / p["goal_usd"] * 100, 1)
|
||||
print(f" ✅ LIVE: {campaign.pledged} / {campaign.goal} | {campaign.backers} backers | {campaign.days_left}d left")
|
||||
else:
|
||||
print(f" ⚠️ FlareSolverr failed, keeping Discover API data")
|
||||
time.sleep(2) # Rate limit
|
||||
|
||||
# Step 5: Save
|
||||
print(f"\n[5] Saving {len(parsed)} campaigns to {OUTPUT_FILE}...")
|
||||
|
||||
output = {
|
||||
"collected_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"total_board_games": len(parsed),
|
||||
"top20_live_fetched": top_n,
|
||||
"campaigns": parsed,
|
||||
}
|
||||
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
json.dump(output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n✅ DONE! {len(parsed)} board game campaigns saved.")
|
||||
print(f" Top 5 by funding:")
|
||||
for p in parsed[:5]:
|
||||
print(f" {p['pledged_display']} / {p['goal_display']} ({p['percent_funded']}%) | {p['name'][:70]}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user