Add residential proxy support to Booking.com scraper

- Route through DataImpulse residential proxy when BOOKING_PROXY_* env
  vars are set (empty = direct connection, unchanged behaviour)
- Sticky one IP per session via DataImpulse sessid; rotate_session()
  burns the IP for a fresh one
- Rotate-on-block: retry a date on a new IP when the page is challenged
  or page 1 renders zero hotels (up to 3 IPs; single attempt sans proxy)
- Persistent, pre-warmed context across dates so cache stays hot
  (~5 MB first page, ~0.3 MB per date after) instead of per-date cold loads
- Block images/media/fonts and third-party ad/consent/analytics hosts to
  cut bandwidth; never touch the anti-bot challenge (awswaf)
- Faster inter-page pacing now that a burned IP is cheap

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/services/scraper_backends/playwright_local.py

@@ -7,12 +7,13 @@ No proxy - direct connection. Suitable for low-volume scraping.
 
 import asyncio
 import logging
+import os
 import random
 import re
 from datetime import date
 from decimal import Decimal, InvalidOperation
 from typing import List, Optional
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse
 
 from playwright.async_api import async_playwright, Browser, BrowserContext, Page
 
@@ -45,6 +46,16 @@ class PlaywrightLocalBackend(ScraperBackend):
     - Warms up via the homepage so search requests carry real session cookies
     - Paginates by clicking "next" rather than deep-linking ?offset=25
     - Random delays, viewport jitter, mouse movement, human-like scroll
+
+    Proxy (optional, via BOOKING_PROXY_* env vars):
+    - Routes through a residential proxy when configured, keeping ONE sticky
+      IP per session so cache stays warm and behaviour looks like one user
+      browsing many dates. rotate_session() burns the IP for a fresh one.
+    - The context is persistent across dates within a session — the first
+      page pays the full ~5 MB cold-cache cost, every date after it is
+      ~0.3 MB on the wire. Rotating the IP resets the cache.
+    - Blocks images/media/fonts and third-party ad/consent/analytics domains
+      to cut bandwidth, but NEVER touches the anti-bot challenge (awswaf).
     """
 
     # A single coherent identity. The UA, the sec-ch-ua hints and the
@@ -62,22 +73,101 @@ class PlaywrightLocalBackend(ScraperBackend):
     }
     HOMEPAGE = "https://www.booking.com/index.en-gb.html"
 
+    # Bandwidth trimming. Heavy asset types the parser never needs, plus
+    # third-party ad/consent/analytics hosts. The anti-bot challenge host
+    # (awswaf) is ALWAYS allowed — a browser that skips it fails silently.
+    BLOCK_RESOURCE_TYPES = {"image", "media", "font"}
+    BLOCK_DOMAINS = (
+        "googlesyndication.com", "pagead2", "google-analytics.com",
+        "googletagmanager.com", "doubleclick.net", "cookielaw.org",
+        "onetrust.com", "accounts.google.com", "connect.facebook.net",
+        "facebook.com", "hotjar.com", "bat.bing.com", "clarity.ms",
+        "criteo.com", "adnxs.com",
+    )
+
     def __init__(self, proxy_config: dict = None):
         """
         Initialize the backend.
 
         Args:
-            proxy_config: Optional proxy configuration (for future use)
+            proxy_config: Optional proxy configuration. Falls back to the
+                BOOKING_PROXY_* environment variables when not supplied.
         """
-        self.proxy_config = proxy_config
+        self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env()
         self._playwright = None
         self._browser: Optional[Browser] = None
+        self._context: Optional[BrowserContext] = None
+        self._session_id: Optional[str] = None
+        if self._proxy_enabled():
+            self._new_session_id()
+            logger.info(
+                f"Proxy enabled via {self.proxy_config['host']}:{self.proxy_config['port']} "
+                f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})"
+            )
+
+    @staticmethod
+    def _proxy_from_env() -> dict:
+        """Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled)."""
+        host = os.getenv("BOOKING_PROXY_HOST", "").strip()
+        if not host:
+            return {}
+        return {
+            "host": host,
+            "port": os.getenv("BOOKING_PROXY_PORT", "823").strip(),
+            "username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
+            "password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
+            "country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
+        }
+
+    def _proxy_enabled(self) -> bool:
+        return bool(self.proxy_config.get("host") and self.proxy_config.get("username"))
+
+    def _new_session_id(self):
+        """Pick a fresh sticky-session id — DataImpulse pins one IP per id."""
+        self._session_id = f"hnf{random.randint(100000, 999999)}"
+
+    def _proxy_launch_arg(self) -> Optional[dict]:
+        """Build Playwright's proxy dict, encoding country + sticky session
+        into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID."""
+        if not self._proxy_enabled():
+            return None
+        username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}"
+        if self._session_id:
+            username += f";sessid.{self._session_id}"
+        return {
+            "server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}",
+            "username": username,
+            "password": self.proxy_config["password"],
+        }
+
+    async def rotate_session(self):
+        """Burn the current proxy IP and warm cache; the next scrape gets a
+        fresh IP + fresh context. No-op when the proxy is disabled."""
+        if not self._proxy_enabled():
+            return
+        old = self._session_id
+        self._new_session_id()
+        logger.info(f"Rotating proxy session {old} -> {self._session_id}")
+        # Session lives in the launch username, so we must drop both.
+        if self._context is not None:
+            try:
+                await self._context.close()
+            except Exception:
+                pass
+            self._context = None
+        if self._browser is not None:
+            try:
+                await self._browser.close()
+            except Exception:
+                pass
+            self._browser = None
 
     async def _ensure_browser(self) -> Browser:
         """Ensure browser is running, start if needed."""
         if self._browser is None or not self._browser.is_connected():
+            if self._playwright is None:
                 self._playwright = await async_playwright().start()
-            self._browser = await self._playwright.chromium.launch(
+            launch_kwargs = dict(
                 # Headful under xvfb — far cleaner fingerprint than headless.
                 headless=False,
                 args=[
@@ -86,12 +176,49 @@ class PlaywrightLocalBackend(ScraperBackend):
                     '--disable-dev-shm-usage',
                     '--disable-features=IsolateOrigins,site-per-process',
                     '--start-maximized',
-                ]
+                ],
             )
+            # Proxy is set at launch (Chromium binds the sticky session, which
+            # lives in the username, at the network layer). Rotating the IP
+            # therefore relaunches the browser — see rotate_session().
+            proxy = self._proxy_launch_arg()
+            if proxy:
+                launch_kwargs['proxy'] = proxy
+            self._browser = await self._playwright.chromium.launch(**launch_kwargs)
         return self._browser
 
-    async def _create_context(self) -> BrowserContext:
+    async def _route_handler(self, route):
-        """Create a browser context with a single coherent Chrome identity."""
+        """Drop heavy assets and third-party ad/consent/analytics requests to
+        save bandwidth. The anti-bot challenge host is always allowed through."""
+        try:
+            req = route.request
+            host = urlparse(req.url).netloc
+            if "awswaf" in host:
+                await route.continue_()
+                return
+            if req.resource_type in self.BLOCK_RESOURCE_TYPES:
+                await route.abort()
+                return
+            if any(d in host for d in self.BLOCK_DOMAINS):
+                await route.abort()
+                return
+            await route.continue_()
+        except Exception:
+            # Never let a routing error kill the page load
+            try:
+                await route.continue_()
+            except Exception:
+                pass
+
+    async def _ensure_context(self) -> BrowserContext:
+        """Return a persistent context with a single coherent Chrome identity.
+
+        The context is reused across every date in a session so the browser
+        cache stays warm (first page ~5 MB, each date after ~0.3 MB). It is
+        warmed up on the homepage exactly once, here at creation time."""
+        if self._context is not None:
+            return self._context
+
         browser = await self._ensure_browser()
         # Small viewport jitter so every session isn't pixel-identical
         width = 1920 + random.randint(-40, 40)
@@ -103,6 +230,16 @@ class PlaywrightLocalBackend(ScraperBackend):
             timezone_id='Europe/London',
             extra_http_headers=self.CLIENT_HINT_HEADERS,
         )
+        await context.route("**/*", self._route_handler)
+
+        # Warm up once per context (per IP) — subsequent date searches reuse
+        # these cookies and cache rather than re-fetching the homepage.
+        page = await context.new_page()
+        await self._prepare_page(page)
+        await self._warm_up(page)
+        await page.close()
+
+        self._context = context
         return context
 
     async def _prepare_page(self, page: Page):
@@ -368,7 +505,9 @@ class PlaywrightLocalBackend(ScraperBackend):
         pages: int = 2
     ) -> ScraperResult:
         """
-        Scrape booking.com location search results.
+        Scrape booking.com location search results, rotating the proxy IP if
+        a session looks soft-blocked (challenge page, or page 1 rendered
+        nothing). With no proxy configured this is a single attempt.
 
         Args:
             location: Location name
@@ -380,26 +519,55 @@ class PlaywrightLocalBackend(ScraperBackend):
         Returns:
             ScraperResult with hotels and rates found
         """
+        max_attempts = 3 if self._proxy_enabled() else 1
+        result = None
+
+        for attempt in range(max_attempts):
+            result = await self._scrape_once(location, check_in, check_out, adults, pages)
+
+            # Soft-block signals: an explicit challenge, or a "successful" load
+            # that yielded zero hotels (page 1 never rendered results).
+            soft_blocked = result.blocked or (result.success and not result.hotels)
+            if not soft_blocked:
+                return result
+
+            if attempt < max_attempts - 1:
+                logger.warning(
+                    f"Attempt {attempt + 1}/{max_attempts} for {check_in} looked blocked "
+                    f"(blocked={result.blocked}, hotels={len(result.hotels)}) — rotating IP"
+                )
+                await self.rotate_session()
+                await asyncio.sleep(random.uniform(1.0, 2.0))
+
+        return result
+
+    async def _scrape_once(
+        self,
+        location: str,
+        check_in: date,
+        check_out: date,
+        adults: int,
+        pages: int
+    ) -> ScraperResult:
+        """A single scrape attempt for one date on the current IP/session."""
         all_hotels = []
         all_rates = []
         seen_hotel_ids = set()
         pages_ok = 0
 
-        context = None
         page = None
 
         try:
-            context = await self._create_context()
+            # Persistent, already-warmed context (cache stays hot across dates)
+            context = await self._ensure_context()
             page = await context.new_page()
             await self._prepare_page(page)
 
-            # Land on the homepage first so the search carries session cookies
-            await self._warm_up(page)
-
             for page_num in range(pages):
-                # Random delay between pages (3-7 seconds)
+                # Short jittered delay between pages. Kept modest because with
+                # rotate-on-block a burned IP costs nothing — throughput wins.
                 if page_num > 0:
-                    delay = random.uniform(3, 7)
+                    delay = random.uniform(1.5, 3.5)
                     logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
                     await asyncio.sleep(delay)
 
@@ -488,10 +656,10 @@ class PlaywrightLocalBackend(ScraperBackend):
                 pages_ok=pages_ok,
             )
         finally:
+            # Close only the page — the context persists across dates so its
+            # cache stays warm. rotate_session()/close() dispose of the context.
             if page:
                 await page.close()
-            if context:
-                await context.close()
 
     async def scrape_hotel_page(
         self,
@@ -515,6 +683,12 @@ class PlaywrightLocalBackend(ScraperBackend):
 
     async def close(self):
         """Clean up browser resources."""
+        if self._context:
+            try:
+                await self._context.close()
+            except Exception:
+                pass
+            self._context = None
         if self._browser:
             await self._browser.close()
             self._browser = None

docker-compose.yml

@@ -9,6 +9,12 @@ services:
       - APP_SLUG=rates
       - SETTINGS_URL=${SETTINGS_URL:-}
       - SETTINGS_SECRET=${SETTINGS_SECRET:-}
+      # Residential proxy for the Booking.com scraper (empty = direct connection).
+      - BOOKING_PROXY_HOST=${BOOKING_PROXY_HOST:-}
+      - BOOKING_PROXY_PORT=${BOOKING_PROXY_PORT:-823}
+      - BOOKING_PROXY_USERNAME=${BOOKING_PROXY_USERNAME:-}
+      - BOOKING_PROXY_PASSWORD=${BOOKING_PROXY_PASSWORD:-}
+      - BOOKING_PROXY_COUNTRY=${BOOKING_PROXY_COUNTRY:-gb}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
       interval: 15s