hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add residential proxy support to Booking.com scraper

Browse source 
- Route through DataImpulse residential proxy when BOOKING_PROXY_* env
  vars are set (empty = direct connection, unchanged behaviour)
- Sticky one IP per session via DataImpulse sessid; rotate_session()
  burns the IP for a fresh one
- Rotate-on-block: retry a date on a new IP when the page is challenged
  or page 1 renders zero hotels (up to 3 IPs; single attempt sans proxy)
- Persistent, pre-warmed context across dates so cache stays hot
  (~5 MB first page, ~0.3 MB per date after) instead of per-date cold loads
- Block images/media/fonts and third-party ad/consent/analytics hosts to
  cut bandwidth; never touch the anti-bot challenge (awswaf)
- Faster inter-page pacing now that a burned IP is cheap

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 20:30:15 +00:00
parent 1743590743
commit 11b995739c
2 changed files with 198 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

208
backend/services/scraper_backends/playwright_local.py
View file

@ -7,12 +7,13 @@ No proxy - direct connection. Suitable for low-volume scraping.
import asyncio
import logging
import os
import random
import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode
from urllib.parse import urlencode, urlparse
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
@ -45,6 +46,16 @@ class PlaywrightLocalBackend(ScraperBackend):
- Warms up via the homepage so search requests carry real session cookies
- Paginates by clicking "next" rather than deep-linking ?offset=25
- Random delays, viewport jitter, mouse movement, human-like scroll
Proxy (optional, via BOOKING_PROXY_* env vars):
- Routes through a residential proxy when configured, keeping ONE sticky
IP per session so cache stays warm and behaviour looks like one user
browsing many dates. rotate_session() burns the IP for a fresh one.
- The context is persistent across dates within a session — the first
page pays the full ~5 MB cold-cache cost, every date after it is
~0.3 MB on the wire. Rotating the IP resets the cache.
- Blocks images/media/fonts and third-party ad/consent/analytics domains
to cut bandwidth, but NEVER touches the anti-bot challenge (awswaf).
"""
# A single coherent identity. The UA, the sec-ch-ua hints and the
@ -62,22 +73,101 @@ class PlaywrightLocalBackend(ScraperBackend):
}
HOMEPAGE = "https://www.booking.com/index.en-gb.html"
# Bandwidth trimming. Heavy asset types the parser never needs, plus
# third-party ad/consent/analytics hosts. The anti-bot challenge host
# (awswaf) is ALWAYS allowed — a browser that skips it fails silently.
BLOCK_RESOURCE_TYPES = {"image", "media", "font"}
BLOCK_DOMAINS = (
"googlesyndication.com", "pagead2", "google-analytics.com",
"googletagmanager.com", "doubleclick.net", "cookielaw.org",
"onetrust.com", "accounts.google.com", "connect.facebook.net",
"facebook.com", "hotjar.com", "bat.bing.com", "clarity.ms",
"criteo.com", "adnxs.com",
)
def __init__(self, proxy_config: dict = None):
"""
Initialize the backend.
Args:
proxy_config: Optional proxy configuration (for future use)
proxy_config: Optional proxy configuration. Falls back to the
BOOKING_PROXY_* environment variables when not supplied.
"""
self.proxy_config = proxy_config
self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env()
self._playwright = None
self._browser: Optional[Browser] = None
self._context: Optional[BrowserContext] = None
self._session_id: Optional[str] = None
if self._proxy_enabled():
self._new_session_id()
logger.info(
f"Proxy enabled via {self.proxy_config['host']}:{self.proxy_config['port']} "
f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})"
)
@staticmethod
def _proxy_from_env() -> dict:
"""Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled)."""
host = os.getenv("BOOKING_PROXY_HOST", "").strip()
if not host:
return {}
return {
"host": host,
"port": os.getenv("BOOKING_PROXY_PORT", "823").strip(),
"username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
"password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
"country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
}
def _proxy_enabled(self) -> bool:
return bool(self.proxy_config.get("host") and self.proxy_config.get("username"))
def _new_session_id(self):
"""Pick a fresh sticky-session id — DataImpulse pins one IP per id."""
self._session_id = f"hnf{random.randint(100000, 999999)}"
def _proxy_launch_arg(self) -> Optional[dict]:
"""Build Playwright's proxy dict, encoding country + sticky session
into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID."""
if not self._proxy_enabled():
return None
username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}"
if self._session_id:
username += f";sessid.{self._session_id}"
return {
"server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}",
"username": username,
"password": self.proxy_config["password"],
}
async def rotate_session(self):
"""Burn the current proxy IP and warm cache; the next scrape gets a
fresh IP + fresh context. No-op when the proxy is disabled."""
if not self._proxy_enabled():
return
old = self._session_id
self._new_session_id()
logger.info(f"Rotating proxy session {old} -> {self._session_id}")
# Session lives in the launch username, so we must drop both.
if self._context is not None:
try:
await self._context.close()
except Exception:
pass
self._context = None
if self._browser is not None:
try:
await self._browser.close()
except Exception:
pass
self._browser = None
async def _ensure_browser(self) -> Browser:
"""Ensure browser is running, start if needed."""
if self._browser is None or not self._browser.is_connected():
if self._playwright is None:
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
launch_kwargs = dict(
# Headful under xvfb — far cleaner fingerprint than headless.
headless=False,
args=[
@ -86,12 +176,49 @@ class PlaywrightLocalBackend(ScraperBackend):
'--disable-dev-shm-usage',
'--disable-features=IsolateOrigins,site-per-process',
'--start-maximized',
]
],
)
# Proxy is set at launch (Chromium binds the sticky session, which
# lives in the username, at the network layer). Rotating the IP
# therefore relaunches the browser — see rotate_session().
proxy = self._proxy_launch_arg()
if proxy:
launch_kwargs['proxy'] = proxy
self._browser = await self._playwright.chromium.launch(**launch_kwargs)
return self._browser
async def _create_context(self) -> BrowserContext:
"""Create a browser context with a single coherent Chrome identity."""
async def _route_handler(self, route):
"""Drop heavy assets and third-party ad/consent/analytics requests to
save bandwidth. The anti-bot challenge host is always allowed through."""
try:
req = route.request
host = urlparse(req.url).netloc
if "awswaf" in host:
await route.continue_()
return
if req.resource_type in self.BLOCK_RESOURCE_TYPES:
await route.abort()
return
if any(d in host for d in self.BLOCK_DOMAINS):
await route.abort()
return
await route.continue_()
except Exception:
# Never let a routing error kill the page load
try:
await route.continue_()
except Exception:
pass
async def _ensure_context(self) -> BrowserContext:
"""Return a persistent context with a single coherent Chrome identity.
The context is reused across every date in a session so the browser
cache stays warm (first page ~5 MB, each date after ~0.3 MB). It is
warmed up on the homepage exactly once, here at creation time."""
if self._context is not None:
return self._context
browser = await self._ensure_browser()
# Small viewport jitter so every session isn't pixel-identical
width = 1920 + random.randint(-40, 40)
@ -103,6 +230,16 @@ class PlaywrightLocalBackend(ScraperBackend):
timezone_id='Europe/London',
extra_http_headers=self.CLIENT_HINT_HEADERS,
)
await context.route("**/*", self._route_handler)
# Warm up once per context (per IP) — subsequent date searches reuse
# these cookies and cache rather than re-fetching the homepage.
page = await context.new_page()
await self._prepare_page(page)
await self._warm_up(page)
await page.close()
self._context = context
return context
async def _prepare_page(self, page: Page):
@ -368,7 +505,9 @@ class PlaywrightLocalBackend(ScraperBackend):
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Scrape booking.com location search results, rotating the proxy IP if
a session looks soft-blocked (challenge page, or page 1 rendered
nothing). With no proxy configured this is a single attempt.
Args:
location: Location name
@ -380,26 +519,55 @@ class PlaywrightLocalBackend(ScraperBackend):
Returns:
ScraperResult with hotels and rates found
"""
max_attempts = 3 if self._proxy_enabled() else 1
result = None
for attempt in range(max_attempts):
result = await self._scrape_once(location, check_in, check_out, adults, pages)
# Soft-block signals: an explicit challenge, or a "successful" load
# that yielded zero hotels (page 1 never rendered results).
soft_blocked = result.blocked or (result.success and not result.hotels)
if not soft_blocked:
return result
if attempt < max_attempts - 1:
logger.warning(
f"Attempt {attempt + 1}/{max_attempts} for {check_in} looked blocked "
f"(blocked={result.blocked}, hotels={len(result.hotels)}) — rotating IP"
)
await self.rotate_session()
await asyncio.sleep(random.uniform(1.0, 2.0))
return result
async def _scrape_once(
self,
location: str,
check_in: date,
check_out: date,
adults: int,
pages: int
) -> ScraperResult:
"""A single scrape attempt for one date on the current IP/session."""
all_hotels = []
all_rates = []
seen_hotel_ids = set()
pages_ok = 0
context = None
page = None
try:
context = await self._create_context()
# Persistent, already-warmed context (cache stays hot across dates)
context = await self._ensure_context()
page = await context.new_page()
await self._prepare_page(page)
# Land on the homepage first so the search carries session cookies
await self._warm_up(page)
for page_num in range(pages):
# Random delay between pages (3-7 seconds)
# Short jittered delay between pages. Kept modest because with
# rotate-on-block a burned IP costs nothing — throughput wins.
if page_num > 0:
delay = random.uniform(3, 7)
delay = random.uniform(1.5, 3.5)
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
await asyncio.sleep(delay)
@ -488,10 +656,10 @@ class PlaywrightLocalBackend(ScraperBackend):
pages_ok=pages_ok,
)
finally:
# Close only the page — the context persists across dates so its
# cache stays warm. rotate_session()/close() dispose of the context.
if page:
await page.close()
if context:
await context.close()
async def scrape_hotel_page(
self,
@ -515,6 +683,12 @@ class PlaywrightLocalBackend(ScraperBackend):
async def close(self):
"""Clean up browser resources."""
if self._context:
try:
await self._context.close()
except Exception:
pass
self._context = None
if self._browser:
await self._browser.close()
self._browser = None

6
docker-compose.yml
View file

@ -9,6 +9,12 @@ services:
- APP_SLUG=rates
- SETTINGS_URL=${SETTINGS_URL:-}
- SETTINGS_SECRET=${SETTINGS_SECRET:-}
# Residential proxy for the Booking.com scraper (empty = direct connection).
- BOOKING_PROXY_HOST=${BOOKING_PROXY_HOST:-}
- BOOKING_PROXY_PORT=${BOOKING_PROXY_PORT:-823}
- BOOKING_PROXY_USERNAME=${BOOKING_PROXY_USERNAME:-}
- BOOKING_PROXY_PASSWORD=${BOOKING_PROXY_PASSWORD:-}
- BOOKING_PROXY_COUNTRY=${BOOKING_PROXY_COUNTRY:-gb}
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 15s