Files
BSN-Chatsystem/scrape_polygon.py

17 lines
483 B
Python

import sys, re, gzip
data = sys.stdin.buffer.read()
if data[:2] == b'\x1f\x8b':
data = gzip.decompress(data)
text = data.decode('utf-8', errors='replace')
links = re.findall(r'<a[^>]*href="(/[^"]+)"[^>]*>([^<]{15,200})</a>', text)
for url, title in links[:30]:
clean = title.strip()
if any(kw in url.lower() for kw in ['board','game','tabletop']):
u = 'https://www.polygon.com' + url
print(f'TITLE: {clean}')
print(f'URL: {u}')
print()