fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)
This commit is contained in:
Hermes Agent
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrape YouTube trending videos for Germany using InnerTube API."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import ssl
|
||||
import sys
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def fetch_url(url, headers=None, data=None, timeout=20):
|
||||
req = urllib.request.Request(url, data=data, headers=headers or {})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
|
||||
return resp.read().decode('utf-8', errors='replace')
|
||||
except Exception as e:
|
||||
print(f"Error fetching {url}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
def get_api_key():
|
||||
html = fetch_url("https://www.youtube.com", headers={
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "de-DE,de;q=0.9"
|
||||
})
|
||||
if not html:
|
||||
return None
|
||||
match = re.search(r'"INNERTUBE_API_KEY"\s*:\s*"([^"]+)"', html)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
def get_trending_videos(api_key):
|
||||
url = f"https://www.youtube.com/youtubei/v1/browse?key={api_key}"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Language": "de-DE,de;q=0.9",
|
||||
"X-YouTube-Client-Name": "1",
|
||||
"X-YouTube-Client-Version": "2.20260609.07.00",
|
||||
"Origin": "https://www.youtube.com",
|
||||
"Referer": "https://www.youtube.com/feed/trending"
|
||||
}
|
||||
body = json.dumps({
|
||||
"context": {
|
||||
"client": {
|
||||
"hl": "de",
|
||||
"gl": "DE",
|
||||
"clientName": "WEB",
|
||||
"clientVersion": "2.20260609.07.00",
|
||||
"userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36,gzip(gfe)",
|
||||
"acceptHeader": "*/*",
|
||||
"utcOffsetMinutes": 120
|
||||
}
|
||||
},
|
||||
"browseId": "FEtrending"
|
||||
}).encode('utf-8')
|
||||
|
||||
resp = fetch_url(url, headers=headers, data=body, timeout=30)
|
||||
if not resp:
|
||||
return None
|
||||
return json.loads(resp)
|
||||
|
||||
def extract_videos(data):
|
||||
videos = []
|
||||
|
||||
def find_video_renderers(obj, path=""):
|
||||
if isinstance(obj, dict):
|
||||
if 'videoRenderer' in obj:
|
||||
vr = obj['videoRenderer']
|
||||
video_id = vr.get('videoId', 'N/A')
|
||||
title_runs = vr.get('title', {}).get('runs', [])
|
||||
title = title_runs[0].get('text', 'N/A') if title_runs else 'N/A'
|
||||
|
||||
owner_runs = vr.get('ownerText', {}).get('runs', [])
|
||||
channel = owner_runs[0].get('text', 'N/A') if owner_runs else 'N/A'
|
||||
|
||||
views_text = vr.get('viewCountText', {})
|
||||
views = views_text.get('simpleText', 'N/A')
|
||||
if views == 'N/A':
|
||||
views_runs = views_text.get('runs', [])
|
||||
views = ''.join(r.get('text', '') for r in views_runs) if views_runs else 'N/A'
|
||||
|
||||
desc_snippet = vr.get('descriptionSnippet', {})
|
||||
desc_runs = desc_snippet.get('runs', [])
|
||||
desc = ''.join(r.get('text', '') for r in desc_runs) if desc_runs else ''
|
||||
|
||||
length_text = vr.get('lengthText', {})
|
||||
length = length_text.get('simpleText', 'N/A')
|
||||
|
||||
videos.append({
|
||||
'videoId': video_id,
|
||||
'title': title,
|
||||
'channel': channel,
|
||||
'views': views,
|
||||
'description_snippet': desc,
|
||||
'length': length
|
||||
})
|
||||
|
||||
for key, value in obj.items():
|
||||
if key != 'videoRenderer':
|
||||
find_video_renderers(value, f"{path}.{key}")
|
||||
|
||||
elif isinstance(obj, list):
|
||||
for i, item in enumerate(obj):
|
||||
find_video_renderers(item, f"{path}[{i}]")
|
||||
|
||||
find_video_renderers(data)
|
||||
return videos
|
||||
|
||||
def get_video_description(api_key, video_id):
|
||||
url = f"https://www.youtube.com/youtubei/v1/player?key={api_key}"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Language": "de-DE,de;q=0.9",
|
||||
"X-YouTube-Client-Name": "1",
|
||||
"X-YouTube-Client-Version": "2.20260609.07.00",
|
||||
"Origin": "https://www.youtube.com",
|
||||
"Referer": f"https://www.youtube.com/watch?v={video_id}"
|
||||
}
|
||||
body = json.dumps({
|
||||
"context": {
|
||||
"client": {
|
||||
"hl": "de",
|
||||
"gl": "DE",
|
||||
"clientName": "WEB",
|
||||
"clientVersion": "2.20260609.07.00",
|
||||
"userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36,gzip(gfe)",
|
||||
"acceptHeader": "*/*",
|
||||
"utcOffsetMinutes": 120
|
||||
}
|
||||
},
|
||||
"videoId": video_id,
|
||||
"racyCheckOk": True
|
||||
}).encode('utf-8')
|
||||
|
||||
resp = fetch_url(url, headers=headers, data=body, timeout=30)
|
||||
if not resp:
|
||||
return None
|
||||
|
||||
try:
|
||||
data = json.loads(resp)
|
||||
except:
|
||||
return None
|
||||
|
||||
video_details = data.get('videoDetails', {})
|
||||
short_desc = video_details.get('shortDescription', '')
|
||||
|
||||
microformat = data.get('microformat', {}).get('playerMicroformatRenderer', {})
|
||||
full_desc = microformat.get('description', {}).get('simpleText', '')
|
||||
if not full_desc:
|
||||
desc_runs = microformat.get('description', {}).get('runs', [])
|
||||
full_desc = ''.join(r.get('text', '') for r in desc_runs)
|
||||
|
||||
return full_desc or short_desc
|
||||
|
||||
def main():
|
||||
print("Step 1: Getting API key...", file=sys.stderr)
|
||||
api_key = get_api_key()
|
||||
if not api_key:
|
||||
print("ERROR: Could not get API key", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"Got API key: {api_key[:15]}...", file=sys.stderr)
|
||||
|
||||
print("Step 2: Fetching trending videos...", file=sys.stderr)
|
||||
data = get_trending_videos(api_key)
|
||||
if not data:
|
||||
print("ERROR: Could not fetch trending data", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
videos = extract_videos(data)
|
||||
print(f"Found {len(videos)} videos", file=sys.stderr)
|
||||
|
||||
top5 = videos[:5]
|
||||
|
||||
print("Step 3: Fetching full descriptions...", file=sys.stderr)
|
||||
results = []
|
||||
for i, v in enumerate(top5):
|
||||
print(f" Fetching description for video {i+1}: {v['title'][:50]}...", file=sys.stderr)
|
||||
full_text = get_video_description(api_key, v['videoId'])
|
||||
|
||||
desc_de = v.get('description_snippet', '')
|
||||
if not desc_de and full_text:
|
||||
sentences = re.split(r'(?<=[.!?])\s+', full_text)
|
||||
desc_de = ' '.join(sentences[:3])
|
||||
|
||||
results.append({
|
||||
"title": v['title'],
|
||||
"channel": v['channel'],
|
||||
"views": v['views'],
|
||||
"description_de": desc_de[:300] if desc_de else "Keine Beschreibung verfügbar.",
|
||||
"source_url": f"https://www.youtube.com/watch?v={v['videoId']}",
|
||||
"full_text": full_text or "N/A",
|
||||
"tags": ["YOUTUBE", "TRENDING", "DEUTSCHLAND"]
|
||||
})
|
||||
|
||||
print(json.dumps(results, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user