diff --git a/backend/Dockerfile b/backend/Dockerfile index 0d810ed..093c7e1 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \ libpq-dev \ postgresql-client \ curl \ + xvfb \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . @@ -19,4 +20,7 @@ COPY . . EXPOSE 8000 -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +# Run under a virtual X display so Chromium launches headful (headful fingerprints +# far cleaner than headless — real GPU strings, plugins, chrome.runtime). +CMD ["xvfb-run", "-a", "--server-args=-screen 0 1920x1080x24", \ + "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/requirements.txt b/backend/requirements.txt index 6f48410..e8d2e27 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,3 +12,4 @@ pydantic-settings==2.1.0 python-dotenv==1.0.0 python-dateutil==2.8.2 playwright>=1.40.0 +playwright-stealth==1.0.6 diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 10ff0e1..2d119a7 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -16,6 +16,11 @@ from urllib.parse import urlencode from playwright.async_api import async_playwright, Browser, BrowserContext, Page +try: + from playwright_stealth import stealth_async +except ImportError: # pragma: no cover - stealth is optional at runtime + stealth_async = None + from .base import ( ScraperBackend, ScraperResult, @@ -31,20 +36,31 @@ class PlaywrightLocalBackend(ScraperBackend): """ Local Playwright backend using Chromium. - Features: - - Rotates user agents - - Random delays between requests - - Mimics human scroll behavior - - Uses data-testid selectors for stability + Stealth measures: + - Runs headful (via xvfb in the container) — headless leaks SwiftShader + WebGL, empty plugins, missing chrome.runtime + - playwright-stealth patches navigator.webdriver, plugins, WebGL vendor + - ONE consistent modern-Chrome identity: UA + matching sec-ch-ua client + hints + platform (a mismatched UA is worse than none) + - Warms up via the homepage so search requests carry real session cookies + - Paginates by clicking "next" rather than deep-linking ?offset=25 + - Random delays, viewport jitter, mouse movement, human-like scroll """ - USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0", - ] + # A single coherent identity. The UA, the sec-ch-ua hints and the + # platform must all agree or the mismatch itself is a bot signal. + CHROME_VERSION = "121" + USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" + ) + CLIENT_HINT_HEADERS = { + "sec-ch-ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "accept-language": "en-GB,en;q=0.9", + } + HOMEPAGE = "https://www.booking.com/index.en-gb.html" def __init__(self, proxy_config: dict = None): """ @@ -62,26 +78,61 @@ class PlaywrightLocalBackend(ScraperBackend): if self._browser is None or not self._browser.is_connected(): self._playwright = await async_playwright().start() self._browser = await self._playwright.chromium.launch( - headless=True, + # Headful under xvfb — far cleaner fingerprint than headless. + headless=False, args=[ '--disable-blink-features=AutomationControlled', '--no-sandbox', '--disable-dev-shm-usage', + '--disable-features=IsolateOrigins,site-per-process', + '--start-maximized', ] ) return self._browser async def _create_context(self) -> BrowserContext: - """Create a new browser context with random user agent.""" + """Create a browser context with a single coherent Chrome identity.""" browser = await self._ensure_browser() + # Small viewport jitter so every session isn't pixel-identical + width = 1920 + random.randint(-40, 40) + height = 1080 + random.randint(-30, 30) context = await browser.new_context( - user_agent=random.choice(self.USER_AGENTS), - viewport={'width': 1920, 'height': 1080}, + user_agent=self.USER_AGENT, + viewport={'width': width, 'height': height}, locale='en-GB', timezone_id='Europe/London', + extra_http_headers=self.CLIENT_HINT_HEADERS, ) return context + async def _prepare_page(self, page: Page): + """Apply stealth patches to a freshly created page.""" + if stealth_async is not None: + try: + await stealth_async(page) + except Exception as e: + logger.warning(f"stealth_async failed, continuing without: {e}") + + async def _warm_up(self, page: Page): + """Visit the homepage first so search requests carry session cookies.""" + try: + await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=30000) + await asyncio.sleep(random.uniform(1.5, 3.0)) + # Dismiss the cookie-consent dialog if present + for sel in ['#onetrust-accept-btn-handler', + '[aria-label="Accept"]', + '[data-testid="cookie-banner-accept"]']: + try: + btn = await page.query_selector(sel) + if btn: + await btn.click() + await asyncio.sleep(random.uniform(0.4, 0.9)) + break + except Exception: + continue + except Exception as e: + logger.warning(f"Homepage warm-up failed, continuing: {e}") + def _build_search_url( self, location: str, @@ -137,21 +188,61 @@ class PlaywrightLocalBackend(ScraperBackend): async def _human_like_scroll(self, page: Page): """Simulate human-like scrolling behavior.""" # Scroll down in increments - for _ in range(3): + for _ in range(random.randint(3, 6)): await page.mouse.wheel(0, random.randint(300, 600)) await asyncio.sleep(random.uniform(0.3, 0.8)) + async def _human_like_mouse(self, page: Page): + """A few random mouse moves — real users generate pointer events.""" + try: + for _ in range(random.randint(2, 4)): + await page.mouse.move(random.randint(100, 1400), random.randint(150, 800)) + await asyncio.sleep(random.uniform(0.1, 0.4)) + except Exception: + pass + + async def _go_to_next_page(self, page: Page) -> bool: + """Click the pagination 'next' control. Returns True if navigation happened.""" + selectors = [ + '[data-testid="pagination-next-btn"]', + 'button[aria-label="Next page"]', + 'a[aria-label="Next page"]', + ] + for sel in selectors: + try: + btn = await page.query_selector(sel) + if btn and await btn.is_enabled(): + await btn.scroll_into_view_if_needed() + await asyncio.sleep(random.uniform(0.3, 0.7)) + await btn.click() + # Results re-render in place; wait for network to settle + try: + await page.wait_for_load_state('networkidle', timeout=15000) + except Exception: + await asyncio.sleep(2) + return True + except Exception as e: + logger.debug(f"Next-page selector {sel} failed: {e}") + continue + return False + async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]: """Extract hotel and rate data from search results page.""" hotels = [] rates = [] - # Wait for property cards - booking.com uses data-testid + # Wait for property cards - booking.com uses data-testid. Retry once + # with a scroll nudge before giving up: cards can lazy-load late. try: await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) except Exception as e: - logger.warning(f"No property cards found: {e}") - return hotels, rates + logger.warning(f"No property cards on first wait, retrying: {e}") + try: + await self._human_like_scroll(page) + await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) + except Exception as e2: + logger.warning(f"No property cards found after retry: {e2}") + return hotels, rates # Get all property cards cards = await page.query_selector_all('[data-testid="property-card"]') @@ -300,6 +391,10 @@ class PlaywrightLocalBackend(ScraperBackend): try: context = await self._create_context() page = await context.new_page() + await self._prepare_page(page) + + # Land on the homepage first so the search carries session cookies + await self._warm_up(page) for page_num in range(pages): # Random delay between pages (3-7 seconds) @@ -308,20 +403,31 @@ class PlaywrightLocalBackend(ScraperBackend): logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}") await asyncio.sleep(delay) - # Build URL with offset for pagination (25 results per page) - url = self._build_search_url( - location, check_in, check_out, adults, - offset=page_num * 25 - ) - - logger.info(f"Scraping page {page_num + 1}: {url}") - page_loaded = True - try: - await page.goto(url, wait_until='networkidle', timeout=30000) - except Exception as e: - logger.warning(f"Page load timeout, continuing: {e}") - page_loaded = False + if page_num == 0: + # Navigate to page 1 by URL + url = self._build_search_url(location, check_in, check_out, adults) + logger.info(f"Scraping page {page_num + 1}: {url}") + try: + await page.goto(url, wait_until='domcontentloaded', timeout=30000) + except Exception as e: + logger.warning(f"Page load timeout, continuing: {e}") + page_loaded = False + else: + # Paginate by clicking "next" like a human — deep-linking + # ?offset=25 is a stronger bot signal and gets throttled + clicked = await self._go_to_next_page(page) + if not clicked: + # Fall back to offset URL if the control isn't found + url = self._build_search_url( + location, check_in, check_out, adults, offset=page_num * 25 + ) + logger.info(f"Next-button not found, offset fallback: {url}") + try: + await page.goto(url, wait_until='domcontentloaded', timeout=30000) + except Exception as e: + logger.warning(f"Page load timeout, continuing: {e}") + page_loaded = False # Check for blocking content = await page.content() @@ -339,10 +445,11 @@ class PlaywrightLocalBackend(ScraperBackend): pages_ok=pages_ok, ) - # Human-like scrolling + # Human-like mouse movement + scrolling + await self._human_like_mouse(page) await self._human_like_scroll(page) - # Extract data + # Extract data (retries the selector once on timeout) hotels, rates = await self._extract_search_results(page, check_in) # A page counts as clean if it loaded fully and parsed.