#!/usr/bin/env python3
"""Scrape YouTube trending videos for Germany using InnerTube API."""

import json
import re
import urllib.request
import urllib.error
import ssl
import sys

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

def fetch_url(url, headers=None, data=None, timeout=20):
    req = urllib.request.Request(url, data=data, headers=headers or {})
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
            return resp.read().decode('utf-8', errors='replace')
    except Exception as e:
        print(f"Error fetching {url}: {e}", file=sys.stderr)
        return None

def get_api_key():
    html = fetch_url("https://www.youtube.com", headers={
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
        "Accept-Language": "de-DE,de;q=0.9"
    })
    if not html:
        return None
    match = re.search(r'"INNERTUBE_API_KEY"\s*:\s*"([^"]+)"', html)
    if match:
        return match.group(1)
    return None

def get_trending_videos(api_key):
    url = f"https://www.youtube.com/youtubei/v1/browse?key={api_key}"
    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
        "Content-Type": "application/json",
        "Accept-Language": "de-DE,de;q=0.9",
        "X-YouTube-Client-Name": "1",
        "X-YouTube-Client-Version": "2.20260609.07.00",
        "Origin": "https://www.youtube.com",
        "Referer": "https://www.youtube.com/feed/trending"
    }
    body = json.dumps({
        "context": {
            "client": {
                "hl": "de",
                "gl": "DE",
                "clientName": "WEB",
                "clientVersion": "2.20260609.07.00",
                "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36,gzip(gfe)",
                "acceptHeader": "*/*",
                "utcOffsetMinutes": 120
            }
        },
        "browseId": "FEtrending"
    }).encode('utf-8')
    
    resp = fetch_url(url, headers=headers, data=body, timeout=30)
    if not resp:
        return None
    return json.loads(resp)

def extract_videos(data):
    videos = []
    
    def find_video_renderers(obj, path=""):
        if isinstance(obj, dict):
            if 'videoRenderer' in obj:
                vr = obj['videoRenderer']
                video_id = vr.get('videoId', 'N/A')
                title_runs = vr.get('title', {}).get('runs', [])
                title = title_runs[0].get('text', 'N/A') if title_runs else 'N/A'
                
                owner_runs = vr.get('ownerText', {}).get('runs', [])
                channel = owner_runs[0].get('text', 'N/A') if owner_runs else 'N/A'
                
                views_text = vr.get('viewCountText', {})
                views = views_text.get('simpleText', 'N/A')
                if views == 'N/A':
                    views_runs = views_text.get('runs', [])
                    views = ''.join(r.get('text', '') for r in views_runs) if views_runs else 'N/A'
                
                desc_snippet = vr.get('descriptionSnippet', {})
                desc_runs = desc_snippet.get('runs', [])
                desc = ''.join(r.get('text', '') for r in desc_runs) if desc_runs else ''
                
                length_text = vr.get('lengthText', {})
                length = length_text.get('simpleText', 'N/A')
                
                videos.append({
                    'videoId': video_id,
                    'title': title,
                    'channel': channel,
                    'views': views,
                    'description_snippet': desc,
                    'length': length
                })
            
            for key, value in obj.items():
                if key != 'videoRenderer':
                    find_video_renderers(value, f"{path}.{key}")
        
        elif isinstance(obj, list):
            for i, item in enumerate(obj):
                find_video_renderers(item, f"{path}[{i}]")
    
    find_video_renderers(data)
    return videos

def get_video_description(api_key, video_id):
    url = f"https://www.youtube.com/youtubei/v1/player?key={api_key}"
    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
        "Content-Type": "application/json",
        "Accept-Language": "de-DE,de;q=0.9",
        "X-YouTube-Client-Name": "1",
        "X-YouTube-Client-Version": "2.20260609.07.00",
        "Origin": "https://www.youtube.com",
        "Referer": f"https://www.youtube.com/watch?v={video_id}"
    }
    body = json.dumps({
        "context": {
            "client": {
                "hl": "de",
                "gl": "DE",
                "clientName": "WEB",
                "clientVersion": "2.20260609.07.00",
                "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36,gzip(gfe)",
                "acceptHeader": "*/*",
                "utcOffsetMinutes": 120
            }
        },
        "videoId": video_id,
        "racyCheckOk": True
    }).encode('utf-8')
    
    resp = fetch_url(url, headers=headers, data=body, timeout=30)
    if not resp:
        return None
    
    try:
        data = json.loads(resp)
    except:
        return None
    
    video_details = data.get('videoDetails', {})
    short_desc = video_details.get('shortDescription', '')
    
    microformat = data.get('microformat', {}).get('playerMicroformatRenderer', {})
    full_desc = microformat.get('description', {}).get('simpleText', '')
    if not full_desc:
        desc_runs = microformat.get('description', {}).get('runs', [])
        full_desc = ''.join(r.get('text', '') for r in desc_runs)
    
    return full_desc or short_desc

def main():
    print("Step 1: Getting API key...", file=sys.stderr)
    api_key = get_api_key()
    if not api_key:
        print("ERROR: Could not get API key", file=sys.stderr)
        sys.exit(1)
    print(f"Got API key: {api_key[:15]}...", file=sys.stderr)
    
    print("Step 2: Fetching trending videos...", file=sys.stderr)
    data = get_trending_videos(api_key)
    if not data:
        print("ERROR: Could not fetch trending data", file=sys.stderr)
        sys.exit(1)
    
    videos = extract_videos(data)
    print(f"Found {len(videos)} videos", file=sys.stderr)
    
    top5 = videos[:5]
    
    print("Step 3: Fetching full descriptions...", file=sys.stderr)
    results = []
    for i, v in enumerate(top5):
        print(f"  Fetching description for video {i+1}: {v['title'][:50]}...", file=sys.stderr)
        full_text = get_video_description(api_key, v['videoId'])
        
        desc_de = v.get('description_snippet', '')
        if not desc_de and full_text:
            sentences = re.split(r'(?<=[.!?])\s+', full_text)
            desc_de = ' '.join(sentences[:3])
        
        results.append({
            "title": v['title'],
            "channel": v['channel'],
            "views": v['views'],
            "description_de": desc_de[:300] if desc_de else "Keine Beschreibung verfügbar.",
            "source_url": f"https://www.youtube.com/watch?v={v['videoId']}",
            "full_text": full_text or "N/A",
            "tags": ["YOUTUBE", "TRENDING", "DEUTSCHLAND"]
        })
    
    print(json.dumps(results, ensure_ascii=False, indent=2))

if __name__ == "__main__":
    main()
