#!/usr/bin/env python3 """Scrape old.reddit.com/r/boardgames with regex-based extraction.""" import json import re import sys import time from datetime import datetime, timezone, date from urllib.request import Request, urlopen USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" BLOCKED_FLAIRS = { "humor", "meme", "memes", "funny", "comc", "check out my collection", "humour", "fluff", "just for fun", "shelfie", "collection", "c.o.m.c", } BLOCKED_TITLE_TERMS = [ "meme", "shitpost", "shelfie saturday", "check out my collection", "comc", "what did you play this week", "wdyp", "circlejerk", ] STICKY_TITLES = [ "daily game recommendations thread", "everything you wanted to know about board games", ] def fetch_html(url): req = Request(url, headers={"User-Agent": USER_AGENT}) with urlopen(req, timeout=30) as resp: return resp.read().decode("utf-8", errors="replace") def is_blocked(flair, title): flair_lower = flair.lower().strip() for blocked in BLOCKED_FLAIRS: if blocked in flair_lower: return True title_lower = title.lower() for term in BLOCKED_TITLE_TERMS: if term in title_lower: return True return False def extract_posts(html): """Extract posts using regex from old.reddit HTML.""" posts = [] things = re.split(r'(
]*>)', html) current_id = None current_attrs = "" for part in things: m = re.match(r'
]*>', part) if m: current_attrs = m.group(1) current_id = m.group(2) continue if not current_id: continue post = { "id": current_id, "title": "", "flair": "", "score": 0, "comments": 0, "timestamp_utc": None, "author": "", "permalink": "", "domain": "", "sticky": "stickied" in current_attrs, } # Timestamp ts_match = re.search(r']*datetime="([^"]+)"', part) if ts_match: try: post["timestamp_utc"] = datetime.fromisoformat(ts_match.group(1)) except: pass # Score (prefer data-score, fallback to score div title) score_match = re.search(r'data-score="(\d+)"', part) if not score_match: score_match = re.search(r'
]*title="(\d+)"', part) if not score_match: score_match = re.search(r'
]*title="(\d+)"', part) if score_match: post["score"] = int(score_match.group(1)) # Comments comments_match = re.search(r'data-comments-count="(\d+)"', part) if not comments_match: comments_match = re.search(r'>(\d+)\s*comments?', part) if comments_match: post["comments"] = int(comments_match.group(1)) # Author from data-author author_match = re.search(r'data-author="([^"]+)"', part) if author_match and author_match.group(1) != "AutoModerator": post["author"] = author_match.group(1) else: # Try from author link text author_match2 = re.search(r']*class="[^"]*\bauthor\b[^"]*"[^>]*>([^<]+)', part) if author_match2: post["author"] = author_match2.group(1) # Domain from data-domain domain_match = re.search(r'data-domain="([^"]+)"', part) if domain_match: post["domain"] = domain_match.group(1) # Permalink pm_match = re.search(r'data-permalink="([^"]+)"', part) if pm_match: post["permalink"] = "https://old.reddit.com" + pm_match.group(1) else: post["permalink"] = f"https://old.reddit.com/r/boardgames/comments/{current_id}/" # Title from title_match = re.search(r']*>\s*(.*?)\s*', part, re.DOTALL) if title_match: title_raw = title_match.group(1) title_raw = title_raw.replace("&", "&").replace("<", "<").replace(">", ">") title_raw = title_raw.replace(" ", " ").replace(""", '"').replace("'", "'") title_raw = re.sub(r'<[^>]+>', '', title_raw) title_raw = " ".join(title_raw.split()) post["title"] = title_raw # Flair flair_match = re.search(r']*>.*?]*>([^<]+)', part) if not flair_match: flair_match = re.search(r']*>([^<]+)', part) if flair_match: post["flair"] = flair_match.group(1).strip() # Skip sticky posts (pinned/megathreads) if post["sticky"]: title_lower = post["title"].lower() for s in STICKY_TITLES: if s in title_lower: continue posts.append(post) current_id = None return posts def filter_posts(posts, target_dates): filtered = [] for post in posts: ts = post.get("timestamp_utc") if ts is None: continue post_date = ts.date() if post_date not in target_dates: continue if is_blocked(post.get("flair", ""), post.get("title", "")): continue # Skip sticky auto-mod posts if post.get("sticky") and "daily" in post.get("title", "").lower(): continue post["date_str"] = ts.strftime("%Y-%m-%d %H:%M UTC") filtered.append(post) return filtered def main(): target_dates = {date(2026, 6, 6), date(2026, 6, 7)} print("=== Scraping old.reddit.com/r/boardgames ===", file=sys.stderr) urls = [ "https://old.reddit.com/r/boardgames/", "https://old.reddit.com/r/boardgames/new/", "https://old.reddit.com/r/boardgames/rising/", ] all_posts = [] for url in urls: print(f"Fetching: {url}", file=sys.stderr) try: html = fetch_html(url) posts = extract_posts(html) print(f" Extracted {len(posts)} posts", file=sys.stderr) existing_ids = {p["id"] for p in all_posts} for p in posts: if p["id"] not in existing_ids: all_posts.append(p) existing_ids.add(p["id"]) except Exception as e: print(f" Error: {e}", file=sys.stderr) time.sleep(1) print(f"\nTotal unique raw posts: {len(all_posts)}", file=sys.stderr) # List all dates found dates_found = set() for p in all_posts: if p.get("timestamp_utc"): dates_found.add(p["timestamp_utc"].date()) print(f"Dates found: {sorted(dates_found)}", file=sys.stderr) filtered = filter_posts(all_posts, target_dates) print(f"After filtering: {len(filtered)} posts", file=sys.stderr) # Sort by score desc filtered.sort(key=lambda p: p.get("score", 0), reverse=True) filtered = filtered[:20] output = [] for i, post in enumerate(filtered, 1): output.append({ "rank": i, "title": post["title"], "score": post["score"], "comments": post["comments"], "flair": post.get("flair", ""), "author": post.get("author", ""), "date": post.get("date_str", ""), "url": post.get("permalink", ""), "domain": post.get("domain", ""), }) # Save JSON with open("/home/hermes/workspace/boardgames_weekend_posts.json", "w") as f: json.dump(output, f, indent=2, ensure_ascii=False) print(json.dumps(output, indent=2, ensure_ascii=False)) # Summary print("\n" + "="*80, file=sys.stderr) print(f"TOP {len(output)} POSTS (Jun 6-7, 2026) from r/boardgames", file=sys.stderr) print("="*80, file=sys.stderr) news_posts = [p for p in output if p["flair"] == "News"] reviews = [p for p in output if p["flair"] == "Review"] questions = [p for p in output if "question" in p["flair"].lower() or "?" in p["title"]] ks_posts = [p for p in output if "kickstarter" in p["title"].lower() or "crowdfunding" in p["title"].lower()] if news_posts: print("\nšŸ“° NEWS:", file=sys.stderr) for p in news_posts: print(f" ā€¢ {p['title']} (šŸ”ŗ{p['score']}, šŸ’¬{p['comments']})", file=sys.stderr) if reviews: print("\nšŸ“ REVIEWS:", file=sys.stderr) for p in reviews: print(f" ā€¢ {p['title']} (šŸ”ŗ{p['score']}, šŸ’¬{p['comments']})", file=sys.stderr) print(f"\nšŸ“Š Total: {len(output)} posts", file=sys.stderr) print(f"šŸ“… Date range: Jun 6-7, 2026", file=sys.stderr) print(f"šŸ“„ JSON saved to: /home/hermes/workspace/boardgames_weekend_posts.json", file=sys.stderr) if __name__ == "__main__": main()