#!/usr/bin/env python3 """Scrape YouTube trending videos for Germany using InnerTube API.""" import json import re import urllib.request import urllib.error import ssl import sys ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE def fetch_url(url, headers=None, data=None, timeout=20): req = urllib.request.Request(url, data=data, headers=headers or {}) try: with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: return resp.read().decode('utf-8', errors='replace') except Exception as e: print(f"Error fetching {url}: {e}", file=sys.stderr) return None def get_api_key(): html = fetch_url("https://www.youtube.com", headers={ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", "Accept-Language": "de-DE,de;q=0.9" }) if not html: return None match = re.search(r'"INNERTUBE_API_KEY"\s*:\s*"([^"]+)"', html) if match: return match.group(1) return None def get_trending_videos(api_key): url = f"https://www.youtube.com/youtubei/v1/browse?key={api_key}" headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", "Content-Type": "application/json", "Accept-Language": "de-DE,de;q=0.9", "X-YouTube-Client-Name": "1", "X-YouTube-Client-Version": "2.20260609.07.00", "Origin": "https://www.youtube.com", "Referer": "https://www.youtube.com/feed/trending" } body = json.dumps({ "context": { "client": { "hl": "de", "gl": "DE", "clientName": "WEB", "clientVersion": "2.20260609.07.00", "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36,gzip(gfe)", "acceptHeader": "*/*", "utcOffsetMinutes": 120 } }, "browseId": "FEtrending" }).encode('utf-8') resp = fetch_url(url, headers=headers, data=body, timeout=30) if not resp: return None return json.loads(resp) def extract_videos(data): videos = [] def find_video_renderers(obj, path=""): if isinstance(obj, dict): if 'videoRenderer' in obj: vr = obj['videoRenderer'] video_id = vr.get('videoId', 'N/A') title_runs = vr.get('title', {}).get('runs', []) title = title_runs[0].get('text', 'N/A') if title_runs else 'N/A' owner_runs = vr.get('ownerText', {}).get('runs', []) channel = owner_runs[0].get('text', 'N/A') if owner_runs else 'N/A' views_text = vr.get('viewCountText', {}) views = views_text.get('simpleText', 'N/A') if views == 'N/A': views_runs = views_text.get('runs', []) views = ''.join(r.get('text', '') for r in views_runs) if views_runs else 'N/A' desc_snippet = vr.get('descriptionSnippet', {}) desc_runs = desc_snippet.get('runs', []) desc = ''.join(r.get('text', '') for r in desc_runs) if desc_runs else '' length_text = vr.get('lengthText', {}) length = length_text.get('simpleText', 'N/A') videos.append({ 'videoId': video_id, 'title': title, 'channel': channel, 'views': views, 'description_snippet': desc, 'length': length }) for key, value in obj.items(): if key != 'videoRenderer': find_video_renderers(value, f"{path}.{key}") elif isinstance(obj, list): for i, item in enumerate(obj): find_video_renderers(item, f"{path}[{i}]") find_video_renderers(data) return videos def get_video_description(api_key, video_id): url = f"https://www.youtube.com/youtubei/v1/player?key={api_key}" headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", "Content-Type": "application/json", "Accept-Language": "de-DE,de;q=0.9", "X-YouTube-Client-Name": "1", "X-YouTube-Client-Version": "2.20260609.07.00", "Origin": "https://www.youtube.com", "Referer": f"https://www.youtube.com/watch?v={video_id}" } body = json.dumps({ "context": { "client": { "hl": "de", "gl": "DE", "clientName": "WEB", "clientVersion": "2.20260609.07.00", "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36,gzip(gfe)", "acceptHeader": "*/*", "utcOffsetMinutes": 120 } }, "videoId": video_id, "racyCheckOk": True }).encode('utf-8') resp = fetch_url(url, headers=headers, data=body, timeout=30) if not resp: return None try: data = json.loads(resp) except: return None video_details = data.get('videoDetails', {}) short_desc = video_details.get('shortDescription', '') microformat = data.get('microformat', {}).get('playerMicroformatRenderer', {}) full_desc = microformat.get('description', {}).get('simpleText', '') if not full_desc: desc_runs = microformat.get('description', {}).get('runs', []) full_desc = ''.join(r.get('text', '') for r in desc_runs) return full_desc or short_desc def main(): print("Step 1: Getting API key...", file=sys.stderr) api_key = get_api_key() if not api_key: print("ERROR: Could not get API key", file=sys.stderr) sys.exit(1) print(f"Got API key: {api_key[:15]}...", file=sys.stderr) print("Step 2: Fetching trending videos...", file=sys.stderr) data = get_trending_videos(api_key) if not data: print("ERROR: Could not fetch trending data", file=sys.stderr) sys.exit(1) videos = extract_videos(data) print(f"Found {len(videos)} videos", file=sys.stderr) top5 = videos[:5] print("Step 3: Fetching full descriptions...", file=sys.stderr) results = [] for i, v in enumerate(top5): print(f" Fetching description for video {i+1}: {v['title'][:50]}...", file=sys.stderr) full_text = get_video_description(api_key, v['videoId']) desc_de = v.get('description_snippet', '') if not desc_de and full_text: sentences = re.split(r'(?<=[.!?])\s+', full_text) desc_de = ' '.join(sentences[:3]) results.append({ "title": v['title'], "channel": v['channel'], "views": v['views'], "description_de": desc_de[:300] if desc_de else "Keine Beschreibung verfügbar.", "source_url": f"https://www.youtube.com/watch?v={v['videoId']}", "full_text": full_text or "N/A", "tags": ["YOUTUBE", "TRENDING", "DEUTSCHLAND"] }) print(json.dumps(results, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()