fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)
This commit is contained in:
Hermes Agent
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrape old.reddit.com/r/boardgames with regex-based extraction."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone, date
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
|
||||
BLOCKED_FLAIRS = {
|
||||
"humor", "meme", "memes", "funny", "comc", "check out my collection",
|
||||
"humour", "fluff", "just for fun", "shelfie", "collection",
|
||||
"c.o.m.c",
|
||||
}
|
||||
|
||||
BLOCKED_TITLE_TERMS = [
|
||||
"meme", "shitpost", "shelfie saturday", "check out my collection",
|
||||
"comc", "what did you play this week", "wdyp", "circlejerk",
|
||||
]
|
||||
|
||||
STICKY_TITLES = [
|
||||
"daily game recommendations thread",
|
||||
"everything you wanted to know about board games",
|
||||
]
|
||||
|
||||
|
||||
def fetch_html(url):
|
||||
req = Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def is_blocked(flair, title):
|
||||
flair_lower = flair.lower().strip()
|
||||
for blocked in BLOCKED_FLAIRS:
|
||||
if blocked in flair_lower:
|
||||
return True
|
||||
title_lower = title.lower()
|
||||
for term in BLOCKED_TITLE_TERMS:
|
||||
if term in title_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def extract_posts(html):
|
||||
"""Extract posts using regex from old.reddit HTML."""
|
||||
posts = []
|
||||
|
||||
things = re.split(r'(<div class="[^"]*?\bthing\b[^"]*?\bid-t3_[a-z0-9]+[^"]*"[^>]*>)', html)
|
||||
|
||||
current_id = None
|
||||
current_attrs = ""
|
||||
|
||||
for part in things:
|
||||
m = re.match(r'<div class="([^"]*?\bthing\b[^"]*?\bid-t3_([a-z0-9]+)[^"]*)"[^>]*>', part)
|
||||
if m:
|
||||
current_attrs = m.group(1)
|
||||
current_id = m.group(2)
|
||||
continue
|
||||
|
||||
if not current_id:
|
||||
continue
|
||||
|
||||
post = {
|
||||
"id": current_id,
|
||||
"title": "",
|
||||
"flair": "",
|
||||
"score": 0,
|
||||
"comments": 0,
|
||||
"timestamp_utc": None,
|
||||
"author": "",
|
||||
"permalink": "",
|
||||
"domain": "",
|
||||
"sticky": "stickied" in current_attrs,
|
||||
}
|
||||
|
||||
# Timestamp
|
||||
ts_match = re.search(r'<time[^>]*datetime="([^"]+)"', part)
|
||||
if ts_match:
|
||||
try:
|
||||
post["timestamp_utc"] = datetime.fromisoformat(ts_match.group(1))
|
||||
except:
|
||||
pass
|
||||
|
||||
# Score (prefer data-score, fallback to score div title)
|
||||
score_match = re.search(r'data-score="(\d+)"', part)
|
||||
if not score_match:
|
||||
score_match = re.search(r'<div class="[^"]*score unvoted[^"]*"[^>]*title="(\d+)"', part)
|
||||
if not score_match:
|
||||
score_match = re.search(r'<div class="[^"]*\bscore\b[^"]*"[^>]*title="(\d+)"', part)
|
||||
if score_match:
|
||||
post["score"] = int(score_match.group(1))
|
||||
|
||||
# Comments
|
||||
comments_match = re.search(r'data-comments-count="(\d+)"', part)
|
||||
if not comments_match:
|
||||
comments_match = re.search(r'>(\d+)\s*comments?</a>', part)
|
||||
if comments_match:
|
||||
post["comments"] = int(comments_match.group(1))
|
||||
|
||||
# Author from data-author
|
||||
author_match = re.search(r'data-author="([^"]+)"', part)
|
||||
if author_match and author_match.group(1) != "AutoModerator":
|
||||
post["author"] = author_match.group(1)
|
||||
else:
|
||||
# Try from author link text
|
||||
author_match2 = re.search(r'<a[^>]*class="[^"]*\bauthor\b[^"]*"[^>]*>([^<]+)</a>', part)
|
||||
if author_match2:
|
||||
post["author"] = author_match2.group(1)
|
||||
|
||||
# Domain from data-domain
|
||||
domain_match = re.search(r'data-domain="([^"]+)"', part)
|
||||
if domain_match:
|
||||
post["domain"] = domain_match.group(1)
|
||||
|
||||
# Permalink
|
||||
pm_match = re.search(r'data-permalink="([^"]+)"', part)
|
||||
if pm_match:
|
||||
post["permalink"] = "https://old.reddit.com" + pm_match.group(1)
|
||||
else:
|
||||
post["permalink"] = f"https://old.reddit.com/r/boardgames/comments/{current_id}/"
|
||||
|
||||
# Title from <a class="...title...">
|
||||
title_match = re.search(r'<a class="[^"]*\btitle\b[^"]*"[^>]*>\s*(.*?)\s*</a>', part, re.DOTALL)
|
||||
if title_match:
|
||||
title_raw = title_match.group(1)
|
||||
title_raw = title_raw.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
title_raw = title_raw.replace(" ", " ").replace(""", '"').replace("'", "'")
|
||||
title_raw = re.sub(r'<[^>]+>', '', title_raw)
|
||||
title_raw = " ".join(title_raw.split())
|
||||
post["title"] = title_raw
|
||||
|
||||
# Flair
|
||||
flair_match = re.search(r'<span class="[^"]*flairrichtext[^"]*"[^>]*>.*?<span[^>]*>([^<]+)</span></span>', part)
|
||||
if not flair_match:
|
||||
flair_match = re.search(r'<span class="[^"]*linkflairlabel[^"]*"[^>]*>([^<]+)</span>', part)
|
||||
if flair_match:
|
||||
post["flair"] = flair_match.group(1).strip()
|
||||
|
||||
# Skip sticky posts (pinned/megathreads)
|
||||
if post["sticky"]:
|
||||
title_lower = post["title"].lower()
|
||||
for s in STICKY_TITLES:
|
||||
if s in title_lower:
|
||||
continue
|
||||
|
||||
posts.append(post)
|
||||
current_id = None
|
||||
|
||||
return posts
|
||||
|
||||
|
||||
def filter_posts(posts, target_dates):
|
||||
filtered = []
|
||||
for post in posts:
|
||||
ts = post.get("timestamp_utc")
|
||||
if ts is None:
|
||||
continue
|
||||
|
||||
post_date = ts.date()
|
||||
if post_date not in target_dates:
|
||||
continue
|
||||
|
||||
if is_blocked(post.get("flair", ""), post.get("title", "")):
|
||||
continue
|
||||
|
||||
# Skip sticky auto-mod posts
|
||||
if post.get("sticky") and "daily" in post.get("title", "").lower():
|
||||
continue
|
||||
|
||||
post["date_str"] = ts.strftime("%Y-%m-%d %H:%M UTC")
|
||||
filtered.append(post)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def main():
|
||||
target_dates = {date(2026, 6, 6), date(2026, 6, 7)}
|
||||
|
||||
print("=== Scraping old.reddit.com/r/boardgames ===", file=sys.stderr)
|
||||
|
||||
urls = [
|
||||
"https://old.reddit.com/r/boardgames/",
|
||||
"https://old.reddit.com/r/boardgames/new/",
|
||||
"https://old.reddit.com/r/boardgames/rising/",
|
||||
]
|
||||
|
||||
all_posts = []
|
||||
for url in urls:
|
||||
print(f"Fetching: {url}", file=sys.stderr)
|
||||
try:
|
||||
html = fetch_html(url)
|
||||
posts = extract_posts(html)
|
||||
print(f" Extracted {len(posts)} posts", file=sys.stderr)
|
||||
|
||||
existing_ids = {p["id"] for p in all_posts}
|
||||
for p in posts:
|
||||
if p["id"] not in existing_ids:
|
||||
all_posts.append(p)
|
||||
existing_ids.add(p["id"])
|
||||
except Exception as e:
|
||||
print(f" Error: {e}", file=sys.stderr)
|
||||
time.sleep(1)
|
||||
|
||||
print(f"\nTotal unique raw posts: {len(all_posts)}", file=sys.stderr)
|
||||
|
||||
# List all dates found
|
||||
dates_found = set()
|
||||
for p in all_posts:
|
||||
if p.get("timestamp_utc"):
|
||||
dates_found.add(p["timestamp_utc"].date())
|
||||
print(f"Dates found: {sorted(dates_found)}", file=sys.stderr)
|
||||
|
||||
filtered = filter_posts(all_posts, target_dates)
|
||||
print(f"After filtering: {len(filtered)} posts", file=sys.stderr)
|
||||
|
||||
# Sort by score desc
|
||||
filtered.sort(key=lambda p: p.get("score", 0), reverse=True)
|
||||
filtered = filtered[:20]
|
||||
|
||||
output = []
|
||||
for i, post in enumerate(filtered, 1):
|
||||
output.append({
|
||||
"rank": i,
|
||||
"title": post["title"],
|
||||
"score": post["score"],
|
||||
"comments": post["comments"],
|
||||
"flair": post.get("flair", ""),
|
||||
"author": post.get("author", ""),
|
||||
"date": post.get("date_str", ""),
|
||||
"url": post.get("permalink", ""),
|
||||
"domain": post.get("domain", ""),
|
||||
})
|
||||
|
||||
# Save JSON
|
||||
with open("/home/hermes/workspace/boardgames_weekend_posts.json", "w") as f:
|
||||
json.dump(output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(json.dumps(output, indent=2, ensure_ascii=False))
|
||||
|
||||
# Summary
|
||||
print("\n" + "="*80, file=sys.stderr)
|
||||
print(f"TOP {len(output)} POSTS (Jun 6-7, 2026) from r/boardgames", file=sys.stderr)
|
||||
print("="*80, file=sys.stderr)
|
||||
|
||||
news_posts = [p for p in output if p["flair"] == "News"]
|
||||
reviews = [p for p in output if p["flair"] == "Review"]
|
||||
questions = [p for p in output if "question" in p["flair"].lower() or "?" in p["title"]]
|
||||
ks_posts = [p for p in output if "kickstarter" in p["title"].lower() or "crowdfunding" in p["title"].lower()]
|
||||
|
||||
if news_posts:
|
||||
print("\n📰 NEWS:", file=sys.stderr)
|
||||
for p in news_posts:
|
||||
print(f" • {p['title']} (🔺{p['score']}, 💬{p['comments']})", file=sys.stderr)
|
||||
if reviews:
|
||||
print("\n📝 REVIEWS:", file=sys.stderr)
|
||||
for p in reviews:
|
||||
print(f" • {p['title']} (🔺{p['score']}, 💬{p['comments']})", file=sys.stderr)
|
||||
|
||||
print(f"\n📊 Total: {len(output)} posts", file=sys.stderr)
|
||||
print(f"📅 Date range: Jun 6-7, 2026", file=sys.stderr)
|
||||
print(f"📄 JSON saved to: /home/hermes/workspace/boardgames_weekend_posts.json", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user