Scraper stealth: headful+xvfb, playwright-stealth, coherent UA, warm-up, click-paginate

Tier 1+2 anti-detection to stop Booking.com throttling page 2:
- Run Chromium headful under xvfb (Dockerfile) — headless leaks SwiftShader
  WebGL, empty plugins, missing chrome.runtime
- playwright-stealth patches navigator.webdriver/plugins/WebGL vendor
- Single coherent Chrome-121 identity: UA + matching sec-ch-ua client hints +
  platform (dropped the Firefox/Safari UA strings — a mismatched UA on a
  Chromium engine is a stronger tell than no rotation)
- Homepage warm-up so search requests carry real session cookies + cookie
  consent dismiss
- Paginate by clicking next (offset= deep-link was the page-2 tell), offset
  URL kept as fallback
- Viewport jitter, mouse movement, longer scroll, selector retry on lazy load

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \
     libpq-dev \
     postgresql-client \
     curl \
+    xvfb \
     && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt .
@@ -19,4 +20,7 @@ COPY . .
 
 EXPOSE 8000
 
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+# Run under a virtual X display so Chromium launches headful (headful fingerprints
+# far cleaner than headless — real GPU strings, plugins, chrome.runtime).
+CMD ["xvfb-run", "-a", "--server-args=-screen 0 1920x1080x24", \
+     "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/requirements.txt

@@ -12,3 +12,4 @@ pydantic-settings==2.1.0
 python-dotenv==1.0.0
 python-dateutil==2.8.2
 playwright>=1.40.0
+playwright-stealth==1.0.6

backend/services/scraper_backends/playwright_local.py

@@ -16,6 +16,11 @@ from urllib.parse import urlencode
 
 from playwright.async_api import async_playwright, Browser, BrowserContext, Page
 
+try:
+    from playwright_stealth import stealth_async
+except ImportError:  # pragma: no cover - stealth is optional at runtime
+    stealth_async = None
+
 from .base import (
     ScraperBackend,
     ScraperResult,
@@ -31,20 +36,31 @@ class PlaywrightLocalBackend(ScraperBackend):
     """
     Local Playwright backend using Chromium.
 
-    Features:
+    Stealth measures:
-    - Rotates user agents
+    - Runs headful (via xvfb in the container) — headless leaks SwiftShader
-    - Random delays between requests
+      WebGL, empty plugins, missing chrome.runtime
-    - Mimics human scroll behavior
+    - playwright-stealth patches navigator.webdriver, plugins, WebGL vendor
-    - Uses data-testid selectors for stability
+    - ONE consistent modern-Chrome identity: UA + matching sec-ch-ua client
+      hints + platform (a mismatched UA is worse than none)
+    - Warms up via the homepage so search requests carry real session cookies
+    - Paginates by clicking "next" rather than deep-linking ?offset=25
+    - Random delays, viewport jitter, mouse movement, human-like scroll
     """
 
-    USER_AGENTS = [
+    # A single coherent identity. The UA, the sec-ch-ua hints and the
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    # platform must all agree or the mismatch itself is a bot signal.
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    CHROME_VERSION = "121"
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    USER_AGENT = (
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
+        "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
-    ]
+    )
+    CLIENT_HINT_HEADERS = {
+        "sec-ch-ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Windows"',
+        "accept-language": "en-GB,en;q=0.9",
+    }
+    HOMEPAGE = "https://www.booking.com/index.en-gb.html"
 
     def __init__(self, proxy_config: dict = None):
         """
@@ -62,26 +78,61 @@ class PlaywrightLocalBackend(ScraperBackend):
         if self._browser is None or not self._browser.is_connected():
             self._playwright = await async_playwright().start()
             self._browser = await self._playwright.chromium.launch(
-                headless=True,
+                # Headful under xvfb — far cleaner fingerprint than headless.
+                headless=False,
                 args=[
                     '--disable-blink-features=AutomationControlled',
                     '--no-sandbox',
                     '--disable-dev-shm-usage',
+                    '--disable-features=IsolateOrigins,site-per-process',
+                    '--start-maximized',
                 ]
             )
         return self._browser
 
     async def _create_context(self) -> BrowserContext:
-        """Create a new browser context with random user agent."""
+        """Create a browser context with a single coherent Chrome identity."""
         browser = await self._ensure_browser()
+        # Small viewport jitter so every session isn't pixel-identical
+        width = 1920 + random.randint(-40, 40)
+        height = 1080 + random.randint(-30, 30)
         context = await browser.new_context(
-            user_agent=random.choice(self.USER_AGENTS),
+            user_agent=self.USER_AGENT,
-            viewport={'width': 1920, 'height': 1080},
+            viewport={'width': width, 'height': height},
             locale='en-GB',
             timezone_id='Europe/London',
+            extra_http_headers=self.CLIENT_HINT_HEADERS,
         )
         return context
 
+    async def _prepare_page(self, page: Page):
+        """Apply stealth patches to a freshly created page."""
+        if stealth_async is not None:
+            try:
+                await stealth_async(page)
+            except Exception as e:
+                logger.warning(f"stealth_async failed, continuing without: {e}")
+
+    async def _warm_up(self, page: Page):
+        """Visit the homepage first so search requests carry session cookies."""
+        try:
+            await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=30000)
+            await asyncio.sleep(random.uniform(1.5, 3.0))
+            # Dismiss the cookie-consent dialog if present
+            for sel in ['#onetrust-accept-btn-handler',
+                        '[aria-label="Accept"]',
+                        '[data-testid="cookie-banner-accept"]']:
+                try:
+                    btn = await page.query_selector(sel)
+                    if btn:
+                        await btn.click()
+                        await asyncio.sleep(random.uniform(0.4, 0.9))
+                        break
+                except Exception:
+                    continue
+        except Exception as e:
+            logger.warning(f"Homepage warm-up failed, continuing: {e}")
+
     def _build_search_url(
         self,
         location: str,
@@ -137,20 +188,60 @@ class PlaywrightLocalBackend(ScraperBackend):
     async def _human_like_scroll(self, page: Page):
         """Simulate human-like scrolling behavior."""
         # Scroll down in increments
-        for _ in range(3):
+        for _ in range(random.randint(3, 6)):
             await page.mouse.wheel(0, random.randint(300, 600))
             await asyncio.sleep(random.uniform(0.3, 0.8))
 
+    async def _human_like_mouse(self, page: Page):
+        """A few random mouse moves — real users generate pointer events."""
+        try:
+            for _ in range(random.randint(2, 4)):
+                await page.mouse.move(random.randint(100, 1400), random.randint(150, 800))
+                await asyncio.sleep(random.uniform(0.1, 0.4))
+        except Exception:
+            pass
+
+    async def _go_to_next_page(self, page: Page) -> bool:
+        """Click the pagination 'next' control. Returns True if navigation happened."""
+        selectors = [
+            '[data-testid="pagination-next-btn"]',
+            'button[aria-label="Next page"]',
+            'a[aria-label="Next page"]',
+        ]
+        for sel in selectors:
+            try:
+                btn = await page.query_selector(sel)
+                if btn and await btn.is_enabled():
+                    await btn.scroll_into_view_if_needed()
+                    await asyncio.sleep(random.uniform(0.3, 0.7))
+                    await btn.click()
+                    # Results re-render in place; wait for network to settle
+                    try:
+                        await page.wait_for_load_state('networkidle', timeout=15000)
+                    except Exception:
+                        await asyncio.sleep(2)
+                    return True
+            except Exception as e:
+                logger.debug(f"Next-page selector {sel} failed: {e}")
+                continue
+        return False
+
     async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
         """Extract hotel and rate data from search results page."""
         hotels = []
         rates = []
 
-        # Wait for property cards - booking.com uses data-testid
+        # Wait for property cards - booking.com uses data-testid. Retry once
+        # with a scroll nudge before giving up: cards can lazy-load late.
         try:
             await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
         except Exception as e:
-            logger.warning(f"No property cards found: {e}")
+            logger.warning(f"No property cards on first wait, retrying: {e}")
+            try:
+                await self._human_like_scroll(page)
+                await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
+            except Exception as e2:
+                logger.warning(f"No property cards found after retry: {e2}")
                 return hotels, rates
 
         # Get all property cards
@@ -300,6 +391,10 @@ class PlaywrightLocalBackend(ScraperBackend):
         try:
             context = await self._create_context()
             page = await context.new_page()
+            await self._prepare_page(page)
+
+            # Land on the homepage first so the search carries session cookies
+            await self._warm_up(page)
 
             for page_num in range(pages):
                 # Random delay between pages (3-7 seconds)
@@ -308,17 +403,28 @@ class PlaywrightLocalBackend(ScraperBackend):
                     logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
                     await asyncio.sleep(delay)
 
-                # Build URL with offset for pagination (25 results per page)
-                url = self._build_search_url(
-                    location, check_in, check_out, adults,
-                    offset=page_num * 25
-                )
-
-                logger.info(f"Scraping page {page_num + 1}: {url}")
-
                 page_loaded = True
+                if page_num == 0:
+                    # Navigate to page 1 by URL
+                    url = self._build_search_url(location, check_in, check_out, adults)
+                    logger.info(f"Scraping page {page_num + 1}: {url}")
                     try:
-                    await page.goto(url, wait_until='networkidle', timeout=30000)
+                        await page.goto(url, wait_until='domcontentloaded', timeout=30000)
+                    except Exception as e:
+                        logger.warning(f"Page load timeout, continuing: {e}")
+                        page_loaded = False
+                else:
+                    # Paginate by clicking "next" like a human — deep-linking
+                    # ?offset=25 is a stronger bot signal and gets throttled
+                    clicked = await self._go_to_next_page(page)
+                    if not clicked:
+                        # Fall back to offset URL if the control isn't found
+                        url = self._build_search_url(
+                            location, check_in, check_out, adults, offset=page_num * 25
+                        )
+                        logger.info(f"Next-button not found, offset fallback: {url}")
+                        try:
+                            await page.goto(url, wait_until='domcontentloaded', timeout=30000)
                         except Exception as e:
                             logger.warning(f"Page load timeout, continuing: {e}")
                             page_loaded = False
@@ -339,10 +445,11 @@ class PlaywrightLocalBackend(ScraperBackend):
                         pages_ok=pages_ok,
                     )
 
-                # Human-like scrolling
+                # Human-like mouse movement + scrolling
+                await self._human_like_mouse(page)
                 await self._human_like_scroll(page)
 
-                # Extract data
+                # Extract data (retries the selector once on timeout)
                 hotels, rates = await self._extract_search_results(page, check_in)
 
                 # A page counts as clean if it loaded fully and parsed.