Files
BSN-Chatsystem/scrape_dicebreaker.py

38 lines
1.3 KiB
Python

import sys, re, gzip
# Read binary data
data = sys.stdin.buffer.read()
# Check if gzip compressed
if data[:2] == b'\x1f\x8b':
data = gzip.decompress(data)
text = data.decode('utf-8', errors='replace')
# Find article titles and links
# Look for link patterns in the HTML
links = re.findall(r'<a[^>]*href="([^"]+)"[^>]*>([^<]{10,200})</a>', text)
for url, title in links[:30]:
clean = title.strip()
if any(kw in url.lower() for kw in ['news', 'game', 'feature', 'review', 'board']):
if not url.startswith('http'):
url = 'https://www.dicebreaker.com' + url
print(f'TITLE: {clean}')
print(f'URL: {url}')
print()
# Also try h2/h3 with links
titles = re.findall(r'<h[23][^>]*>(.*?)</h[23]>', text, re.DOTALL)
for t in titles[:20]:
clean = re.sub(r'<[^>]+>', '', t).strip()
link = re.search(r'href="([^"]+)"', t)
if clean and len(clean) > 10 and any(kw in clean.lower() for kw in ['game','board','play','card','dice','tabletop','news','release','announc','new','review','launch']):
url = ''
if link:
u = link.group(1)
url = 'https://www.dicebreaker.com' + u if u.startswith('/') else u
print(f'H-TITLE: {clean}')
if url:
print(f'URL: {url}')
print()