fix: QC-Ratings aus 7 älteren Artikeln entfernt (CHECK 10)

2026-06-23 23:47:51 +02:00
parent e73e0e88de
commit 4d90e4249d
656 changed files with 2602398 additions and 3 deletions
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Scrape mobile.de for Mazda 2 listings near Mönchengladbach using Playwright + proxy"""
+import asyncio, json, re, sys
+from playwright.async_api import async_playwright
+
+MG_PLZ = "41061"
+BUDGET = 10000
+MAX_KM = 120000
+MIN_YEAR = 2014
+
+# mobile.de URL with filters
+URL = (
+    f"https://suchen.mobile.de/fahrzeuge/search.html"
+    f"?dam=0&isSearchRequest=true&ms=1200;3;;&pMax={BUDGET}"
+    f"&mlMin={MIN_YEAR}&mlMax:{MAX_KM}&sf={MG_PLZ}&vc=Car&od=down&sb=rel"
+)
+
+PROXY = {
+    "server": "http://geo.iproyal.com:12321",
+    "username": "8B7M8nJOfXTNHPUw",
+    "password": "lR1VidJhN5RrRbeK_region-europe"
+}
+
+async def main():
+    listings = []
+    
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(
+            headless=True,
+            args=['--no-sandbox', '--disable-setuid-sandbox']
+        )
+        context = await browser.new_context(
+            proxy=PROXY,
+            user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+            locale='de-DE',
+            timezone_id='Europe/Berlin'
+        )
+        page = await context.new_page()
+        
+        try:
+            print(f"Loading {URL}")
+            await page.goto(URL, wait_until='networkidle', timeout=30000)
+            
+            content = await page.content()
+            print(f"Page loaded: {len(content)} bytes")
+            
+            if 'Zugriff verweigert' in content or 'Access denied' in content:
+                print("BLOCKED - access denied with proxy too")
+                await browser.close()
+                return []
+            
+            # Wait for results to load
+            try:
+                await page.wait_for_selector('[data-testid="result-list"], .result-list, .cfyXj, article', timeout=10000)
+            except:
+                print("No results selector found, trying to extract anyway...")
+            
+            # Extract listings from page
+            content = await page.content()
+            
+            # Try to find structured data via JSON-LD or embedded data
+            # mobile.de embeds listing data in <script> tags or in the HTML
+            
+            # Method 1: Look for data attributes with listing info
+            listing_blocks = re.findall(
+                r'<article[^>]*>(.*?)</article>',
+                content, re.DOTALL
+            )
+            print(f"Found {len(listing_blocks)} article blocks")
+            
+            if not listing_blocks:
+                # Try alternative selectors
+                listing_blocks = re.findall(
+                    r'<(?:div|article)[^>]*class="[^"]*?(?:result-item|listing-item|cBoxBody)[^"]*"[^>]*>(.*?)</(?:div|article)>',
+                    content, re.DOTALL
+                )
+                print(f"Found {len(listing_blocks)} result-item blocks")
+            
+            for i, block in enumerate(listing_blocks[:25]):
+                # Extract title
+                title_m = re.search(r'<h[23][^>]*>(.*?)</h[23]>', block, re.DOTALL)
+                title = re.sub(r'<[^>]+>', '', title_m.group(1)).strip() if title_m else '?'
+                
+                # Extract price
+                price_m = re.search(r'(\d{1,2}[\.\d]*\s*€)', block)
+                price = price_m.group(1) if price_m else '?'
+                
+                # Extract mileage
+                km_m = re.search(r'(\d{1,3}[\.\d]*\s*km)', block, re.IGNORECASE)
+                km = km_m.group(1) if km_m else '?'
+                
+                # Extract year
+                year_m = re.search(r'(EZ\s*\d{1,2}/\d{4}|(?:20[12]\d{1}))', block)
+                year = year_m.group(1) if year_m else '?'
+                
+                # Extract location
+                loc_m = re.search(r'(?:4[0-9]{4}\s[A-Za-zäöüß\s-]+)', block)
+                loc = loc_m.group(0) if loc_m else '?'
+                
+                # Extract link
+                link_m = re.search(r'href="(/fahrzeuge/detail/[^"]+)"', block)
+                link = f"https://suchen.mobile.de{link_m.group(1)}" if link_m else '?'
+                
+                listings.append({
+                    'title': title,
+                    'price': price,
+                    'km': km,
+                    'year': year,
+                    'location': loc,
+                    'link': link
+                })
+            
+            # If no blocks found, try extracting from full page text
+            if not listings:
+                # Try to find price/km/year patterns in the whole page
+                price_kms = re.findall(r'(\d{1,2}[\.\d]*\s*€)\s*.*?(\d{1,3}[\.\d]*\s*km)', content, re.DOTALL)
+                print(f"Found {len(price_kms)} price-km pairs in full text")
+                for pk in price_kms[:15]:
+                    print(f"  {pk[0]} | {pk[1]}")
+            
+            # Print results
+            print(f"\n=== Extracted {len(listings)} listings ===")
+            for i, l in enumerate(listings):
+                print(f"{i+1}. {l['title']}")
+                print(f"   {l['price']} | {l['km']} | {l['year']} | {l['location']}")
+                print(f"   {l['link']}")
+                print()
+            
+        except Exception as e:
+            print(f"Error: {e}")
+            import traceback
+            traceback.print_exc()
+        finally:
+            await browser.close()
+    
+    return listings
+
+if __name__ == "__main__":
+    listings = asyncio.run(main())
+    # Write JSON output
+    with open("/tmp/mazda2_listings.json", "w") as f:
+        json.dump(listings, f, indent=2, ensure_ascii=False)
+    print(f"\nSaved {len(listings)} listings to /tmp/mazda2_listings.json")