Files
BSN-Chatsystem/scrape_bgg.py
T

32 lines
930 B
Python

import sys, re, gzip
data = sys.stdin.buffer.read()
if data[:2] == b'\x1f\x8b':
data = gzip.decompress(data)
text = data.decode('utf-8', errors='replace')
# Find article titles in BGG
titles = re.findall(r'<h[23][^>]*>(.*?)</h[23]>', text, re.DOTALL)
for t in titles[:25]:
clean = re.sub(r'<[^>]+>', '', t).strip()
link = re.search(r'href="([^"]+)"', t)
if clean and len(clean) > 10:
url = ''
if link:
u = link.group(1)
url = 'https://boardgamegeek.com' + u if u.startswith('/') else u
print(f'TITLE: {clean}')
if url:
print(f'URL: {url}')
print()
# Also try article/link patterns
links = re.findall(r'href="(/blogpost/[^"]+)"[^>]*>([^<]+)</a>', text)
for url, title in links[:20]:
clean = title.strip()
if len(clean) > 10:
print(f'BLOG: {clean}')
print(f'URL: https://boardgamegeek.com{url}')
print()