From 11b995739c32c5f4d18adb003777555346670900 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Sun, 5 Jul 2026 20:30:15 +0000 Subject: [PATCH] Add residential proxy support to Booking.com scraper - Route through DataImpulse residential proxy when BOOKING_PROXY_* env vars are set (empty = direct connection, unchanged behaviour) - Sticky one IP per session via DataImpulse sessid; rotate_session() burns the IP for a fresh one - Rotate-on-block: retry a date on a new IP when the page is challenged or page 1 renders zero hotels (up to 3 IPs; single attempt sans proxy) - Persistent, pre-warmed context across dates so cache stays hot (~5 MB first page, ~0.3 MB per date after) instead of per-date cold loads - Block images/media/fonts and third-party ad/consent/analytics hosts to cut bandwidth; never touch the anti-bot challenge (awswaf) - Faster inter-page pacing now that a burned IP is cheap Co-Authored-By: Claude Fable 5 --- .../scraper_backends/playwright_local.py | 210 ++++++++++++++++-- docker-compose.yml | 6 + 2 files changed, 198 insertions(+), 18 deletions(-) diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 2d119a7..7802843 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -7,12 +7,13 @@ No proxy - direct connection. Suitable for low-volume scraping. import asyncio import logging +import os import random import re from datetime import date from decimal import Decimal, InvalidOperation from typing import List, Optional -from urllib.parse import urlencode +from urllib.parse import urlencode, urlparse from playwright.async_api import async_playwright, Browser, BrowserContext, Page @@ -45,6 +46,16 @@ class PlaywrightLocalBackend(ScraperBackend): - Warms up via the homepage so search requests carry real session cookies - Paginates by clicking "next" rather than deep-linking ?offset=25 - Random delays, viewport jitter, mouse movement, human-like scroll + + Proxy (optional, via BOOKING_PROXY_* env vars): + - Routes through a residential proxy when configured, keeping ONE sticky + IP per session so cache stays warm and behaviour looks like one user + browsing many dates. rotate_session() burns the IP for a fresh one. + - The context is persistent across dates within a session — the first + page pays the full ~5 MB cold-cache cost, every date after it is + ~0.3 MB on the wire. Rotating the IP resets the cache. + - Blocks images/media/fonts and third-party ad/consent/analytics domains + to cut bandwidth, but NEVER touches the anti-bot challenge (awswaf). """ # A single coherent identity. The UA, the sec-ch-ua hints and the @@ -62,22 +73,101 @@ class PlaywrightLocalBackend(ScraperBackend): } HOMEPAGE = "https://www.booking.com/index.en-gb.html" + # Bandwidth trimming. Heavy asset types the parser never needs, plus + # third-party ad/consent/analytics hosts. The anti-bot challenge host + # (awswaf) is ALWAYS allowed — a browser that skips it fails silently. + BLOCK_RESOURCE_TYPES = {"image", "media", "font"} + BLOCK_DOMAINS = ( + "googlesyndication.com", "pagead2", "google-analytics.com", + "googletagmanager.com", "doubleclick.net", "cookielaw.org", + "onetrust.com", "accounts.google.com", "connect.facebook.net", + "facebook.com", "hotjar.com", "bat.bing.com", "clarity.ms", + "criteo.com", "adnxs.com", + ) + def __init__(self, proxy_config: dict = None): """ Initialize the backend. Args: - proxy_config: Optional proxy configuration (for future use) + proxy_config: Optional proxy configuration. Falls back to the + BOOKING_PROXY_* environment variables when not supplied. """ - self.proxy_config = proxy_config + self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env() self._playwright = None self._browser: Optional[Browser] = None + self._context: Optional[BrowserContext] = None + self._session_id: Optional[str] = None + if self._proxy_enabled(): + self._new_session_id() + logger.info( + f"Proxy enabled via {self.proxy_config['host']}:{self.proxy_config['port']} " + f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})" + ) + + @staticmethod + def _proxy_from_env() -> dict: + """Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled).""" + host = os.getenv("BOOKING_PROXY_HOST", "").strip() + if not host: + return {} + return { + "host": host, + "port": os.getenv("BOOKING_PROXY_PORT", "823").strip(), + "username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(), + "password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(), + "country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(), + } + + def _proxy_enabled(self) -> bool: + return bool(self.proxy_config.get("host") and self.proxy_config.get("username")) + + def _new_session_id(self): + """Pick a fresh sticky-session id — DataImpulse pins one IP per id.""" + self._session_id = f"hnf{random.randint(100000, 999999)}" + + def _proxy_launch_arg(self) -> Optional[dict]: + """Build Playwright's proxy dict, encoding country + sticky session + into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID.""" + if not self._proxy_enabled(): + return None + username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}" + if self._session_id: + username += f";sessid.{self._session_id}" + return { + "server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}", + "username": username, + "password": self.proxy_config["password"], + } + + async def rotate_session(self): + """Burn the current proxy IP and warm cache; the next scrape gets a + fresh IP + fresh context. No-op when the proxy is disabled.""" + if not self._proxy_enabled(): + return + old = self._session_id + self._new_session_id() + logger.info(f"Rotating proxy session {old} -> {self._session_id}") + # Session lives in the launch username, so we must drop both. + if self._context is not None: + try: + await self._context.close() + except Exception: + pass + self._context = None + if self._browser is not None: + try: + await self._browser.close() + except Exception: + pass + self._browser = None async def _ensure_browser(self) -> Browser: """Ensure browser is running, start if needed.""" if self._browser is None or not self._browser.is_connected(): - self._playwright = await async_playwright().start() - self._browser = await self._playwright.chromium.launch( + if self._playwright is None: + self._playwright = await async_playwright().start() + launch_kwargs = dict( # Headful under xvfb — far cleaner fingerprint than headless. headless=False, args=[ @@ -86,12 +176,49 @@ class PlaywrightLocalBackend(ScraperBackend): '--disable-dev-shm-usage', '--disable-features=IsolateOrigins,site-per-process', '--start-maximized', - ] + ], ) + # Proxy is set at launch (Chromium binds the sticky session, which + # lives in the username, at the network layer). Rotating the IP + # therefore relaunches the browser — see rotate_session(). + proxy = self._proxy_launch_arg() + if proxy: + launch_kwargs['proxy'] = proxy + self._browser = await self._playwright.chromium.launch(**launch_kwargs) return self._browser - async def _create_context(self) -> BrowserContext: - """Create a browser context with a single coherent Chrome identity.""" + async def _route_handler(self, route): + """Drop heavy assets and third-party ad/consent/analytics requests to + save bandwidth. The anti-bot challenge host is always allowed through.""" + try: + req = route.request + host = urlparse(req.url).netloc + if "awswaf" in host: + await route.continue_() + return + if req.resource_type in self.BLOCK_RESOURCE_TYPES: + await route.abort() + return + if any(d in host for d in self.BLOCK_DOMAINS): + await route.abort() + return + await route.continue_() + except Exception: + # Never let a routing error kill the page load + try: + await route.continue_() + except Exception: + pass + + async def _ensure_context(self) -> BrowserContext: + """Return a persistent context with a single coherent Chrome identity. + + The context is reused across every date in a session so the browser + cache stays warm (first page ~5 MB, each date after ~0.3 MB). It is + warmed up on the homepage exactly once, here at creation time.""" + if self._context is not None: + return self._context + browser = await self._ensure_browser() # Small viewport jitter so every session isn't pixel-identical width = 1920 + random.randint(-40, 40) @@ -103,6 +230,16 @@ class PlaywrightLocalBackend(ScraperBackend): timezone_id='Europe/London', extra_http_headers=self.CLIENT_HINT_HEADERS, ) + await context.route("**/*", self._route_handler) + + # Warm up once per context (per IP) — subsequent date searches reuse + # these cookies and cache rather than re-fetching the homepage. + page = await context.new_page() + await self._prepare_page(page) + await self._warm_up(page) + await page.close() + + self._context = context return context async def _prepare_page(self, page: Page): @@ -368,7 +505,9 @@ class PlaywrightLocalBackend(ScraperBackend): pages: int = 2 ) -> ScraperResult: """ - Scrape booking.com location search results. + Scrape booking.com location search results, rotating the proxy IP if + a session looks soft-blocked (challenge page, or page 1 rendered + nothing). With no proxy configured this is a single attempt. Args: location: Location name @@ -380,26 +519,55 @@ class PlaywrightLocalBackend(ScraperBackend): Returns: ScraperResult with hotels and rates found """ + max_attempts = 3 if self._proxy_enabled() else 1 + result = None + + for attempt in range(max_attempts): + result = await self._scrape_once(location, check_in, check_out, adults, pages) + + # Soft-block signals: an explicit challenge, or a "successful" load + # that yielded zero hotels (page 1 never rendered results). + soft_blocked = result.blocked or (result.success and not result.hotels) + if not soft_blocked: + return result + + if attempt < max_attempts - 1: + logger.warning( + f"Attempt {attempt + 1}/{max_attempts} for {check_in} looked blocked " + f"(blocked={result.blocked}, hotels={len(result.hotels)}) — rotating IP" + ) + await self.rotate_session() + await asyncio.sleep(random.uniform(1.0, 2.0)) + + return result + + async def _scrape_once( + self, + location: str, + check_in: date, + check_out: date, + adults: int, + pages: int + ) -> ScraperResult: + """A single scrape attempt for one date on the current IP/session.""" all_hotels = [] all_rates = [] seen_hotel_ids = set() pages_ok = 0 - context = None page = None try: - context = await self._create_context() + # Persistent, already-warmed context (cache stays hot across dates) + context = await self._ensure_context() page = await context.new_page() await self._prepare_page(page) - # Land on the homepage first so the search carries session cookies - await self._warm_up(page) - for page_num in range(pages): - # Random delay between pages (3-7 seconds) + # Short jittered delay between pages. Kept modest because with + # rotate-on-block a burned IP costs nothing — throughput wins. if page_num > 0: - delay = random.uniform(3, 7) + delay = random.uniform(1.5, 3.5) logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}") await asyncio.sleep(delay) @@ -488,10 +656,10 @@ class PlaywrightLocalBackend(ScraperBackend): pages_ok=pages_ok, ) finally: + # Close only the page — the context persists across dates so its + # cache stays warm. rotate_session()/close() dispose of the context. if page: await page.close() - if context: - await context.close() async def scrape_hotel_page( self, @@ -515,6 +683,12 @@ class PlaywrightLocalBackend(ScraperBackend): async def close(self): """Clean up browser resources.""" + if self._context: + try: + await self._context.close() + except Exception: + pass + self._context = None if self._browser: await self._browser.close() self._browser = None diff --git a/docker-compose.yml b/docker-compose.yml index 0fbd5fd..19d0130 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,12 @@ services: - APP_SLUG=rates - SETTINGS_URL=${SETTINGS_URL:-} - SETTINGS_SECRET=${SETTINGS_SECRET:-} + # Residential proxy for the Booking.com scraper (empty = direct connection). + - BOOKING_PROXY_HOST=${BOOKING_PROXY_HOST:-} + - BOOKING_PROXY_PORT=${BOOKING_PROXY_PORT:-823} + - BOOKING_PROXY_USERNAME=${BOOKING_PROXY_USERNAME:-} + - BOOKING_PROXY_PASSWORD=${BOOKING_PROXY_PASSWORD:-} + - BOOKING_PROXY_COUNTRY=${BOOKING_PROXY_COUNTRY:-gb} healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 15s