Add residential proxy support to Booking.com scraper
- Route through DataImpulse residential proxy when BOOKING_PROXY_* env vars are set (empty = direct connection, unchanged behaviour) - Sticky one IP per session via DataImpulse sessid; rotate_session() burns the IP for a fresh one - Rotate-on-block: retry a date on a new IP when the page is challenged or page 1 renders zero hotels (up to 3 IPs; single attempt sans proxy) - Persistent, pre-warmed context across dates so cache stays hot (~5 MB first page, ~0.3 MB per date after) instead of per-date cold loads - Block images/media/fonts and third-party ad/consent/analytics hosts to cut bandwidth; never touch the anti-bot challenge (awswaf) - Faster inter-page pacing now that a burned IP is cheap Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
1743590743
commit
11b995739c
2 changed files with 198 additions and 18 deletions
|
|
@ -7,12 +7,13 @@ No proxy - direct connection. Suitable for low-volume scraping.
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
from datetime import date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
||||
|
||||
|
|
@ -45,6 +46,16 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
- Warms up via the homepage so search requests carry real session cookies
|
||||
- Paginates by clicking "next" rather than deep-linking ?offset=25
|
||||
- Random delays, viewport jitter, mouse movement, human-like scroll
|
||||
|
||||
Proxy (optional, via BOOKING_PROXY_* env vars):
|
||||
- Routes through a residential proxy when configured, keeping ONE sticky
|
||||
IP per session so cache stays warm and behaviour looks like one user
|
||||
browsing many dates. rotate_session() burns the IP for a fresh one.
|
||||
- The context is persistent across dates within a session — the first
|
||||
page pays the full ~5 MB cold-cache cost, every date after it is
|
||||
~0.3 MB on the wire. Rotating the IP resets the cache.
|
||||
- Blocks images/media/fonts and third-party ad/consent/analytics domains
|
||||
to cut bandwidth, but NEVER touches the anti-bot challenge (awswaf).
|
||||
"""
|
||||
|
||||
# A single coherent identity. The UA, the sec-ch-ua hints and the
|
||||
|
|
@ -62,22 +73,101 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
}
|
||||
HOMEPAGE = "https://www.booking.com/index.en-gb.html"
|
||||
|
||||
# Bandwidth trimming. Heavy asset types the parser never needs, plus
|
||||
# third-party ad/consent/analytics hosts. The anti-bot challenge host
|
||||
# (awswaf) is ALWAYS allowed — a browser that skips it fails silently.
|
||||
BLOCK_RESOURCE_TYPES = {"image", "media", "font"}
|
||||
BLOCK_DOMAINS = (
|
||||
"googlesyndication.com", "pagead2", "google-analytics.com",
|
||||
"googletagmanager.com", "doubleclick.net", "cookielaw.org",
|
||||
"onetrust.com", "accounts.google.com", "connect.facebook.net",
|
||||
"facebook.com", "hotjar.com", "bat.bing.com", "clarity.ms",
|
||||
"criteo.com", "adnxs.com",
|
||||
)
|
||||
|
||||
def __init__(self, proxy_config: dict = None):
|
||||
"""
|
||||
Initialize the backend.
|
||||
|
||||
Args:
|
||||
proxy_config: Optional proxy configuration (for future use)
|
||||
proxy_config: Optional proxy configuration. Falls back to the
|
||||
BOOKING_PROXY_* environment variables when not supplied.
|
||||
"""
|
||||
self.proxy_config = proxy_config
|
||||
self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env()
|
||||
self._playwright = None
|
||||
self._browser: Optional[Browser] = None
|
||||
self._context: Optional[BrowserContext] = None
|
||||
self._session_id: Optional[str] = None
|
||||
if self._proxy_enabled():
|
||||
self._new_session_id()
|
||||
logger.info(
|
||||
f"Proxy enabled via {self.proxy_config['host']}:{self.proxy_config['port']} "
|
||||
f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _proxy_from_env() -> dict:
|
||||
"""Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled)."""
|
||||
host = os.getenv("BOOKING_PROXY_HOST", "").strip()
|
||||
if not host:
|
||||
return {}
|
||||
return {
|
||||
"host": host,
|
||||
"port": os.getenv("BOOKING_PROXY_PORT", "823").strip(),
|
||||
"username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
|
||||
"password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
|
||||
"country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
|
||||
}
|
||||
|
||||
def _proxy_enabled(self) -> bool:
|
||||
return bool(self.proxy_config.get("host") and self.proxy_config.get("username"))
|
||||
|
||||
def _new_session_id(self):
|
||||
"""Pick a fresh sticky-session id — DataImpulse pins one IP per id."""
|
||||
self._session_id = f"hnf{random.randint(100000, 999999)}"
|
||||
|
||||
def _proxy_launch_arg(self) -> Optional[dict]:
|
||||
"""Build Playwright's proxy dict, encoding country + sticky session
|
||||
into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID."""
|
||||
if not self._proxy_enabled():
|
||||
return None
|
||||
username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}"
|
||||
if self._session_id:
|
||||
username += f";sessid.{self._session_id}"
|
||||
return {
|
||||
"server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}",
|
||||
"username": username,
|
||||
"password": self.proxy_config["password"],
|
||||
}
|
||||
|
||||
async def rotate_session(self):
|
||||
"""Burn the current proxy IP and warm cache; the next scrape gets a
|
||||
fresh IP + fresh context. No-op when the proxy is disabled."""
|
||||
if not self._proxy_enabled():
|
||||
return
|
||||
old = self._session_id
|
||||
self._new_session_id()
|
||||
logger.info(f"Rotating proxy session {old} -> {self._session_id}")
|
||||
# Session lives in the launch username, so we must drop both.
|
||||
if self._context is not None:
|
||||
try:
|
||||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._context = None
|
||||
if self._browser is not None:
|
||||
try:
|
||||
await self._browser.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._browser = None
|
||||
|
||||
async def _ensure_browser(self) -> Browser:
|
||||
"""Ensure browser is running, start if needed."""
|
||||
if self._browser is None or not self._browser.is_connected():
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
if self._playwright is None:
|
||||
self._playwright = await async_playwright().start()
|
||||
launch_kwargs = dict(
|
||||
# Headful under xvfb — far cleaner fingerprint than headless.
|
||||
headless=False,
|
||||
args=[
|
||||
|
|
@ -86,12 +176,49 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
'--disable-dev-shm-usage',
|
||||
'--disable-features=IsolateOrigins,site-per-process',
|
||||
'--start-maximized',
|
||||
]
|
||||
],
|
||||
)
|
||||
# Proxy is set at launch (Chromium binds the sticky session, which
|
||||
# lives in the username, at the network layer). Rotating the IP
|
||||
# therefore relaunches the browser — see rotate_session().
|
||||
proxy = self._proxy_launch_arg()
|
||||
if proxy:
|
||||
launch_kwargs['proxy'] = proxy
|
||||
self._browser = await self._playwright.chromium.launch(**launch_kwargs)
|
||||
return self._browser
|
||||
|
||||
async def _create_context(self) -> BrowserContext:
|
||||
"""Create a browser context with a single coherent Chrome identity."""
|
||||
async def _route_handler(self, route):
|
||||
"""Drop heavy assets and third-party ad/consent/analytics requests to
|
||||
save bandwidth. The anti-bot challenge host is always allowed through."""
|
||||
try:
|
||||
req = route.request
|
||||
host = urlparse(req.url).netloc
|
||||
if "awswaf" in host:
|
||||
await route.continue_()
|
||||
return
|
||||
if req.resource_type in self.BLOCK_RESOURCE_TYPES:
|
||||
await route.abort()
|
||||
return
|
||||
if any(d in host for d in self.BLOCK_DOMAINS):
|
||||
await route.abort()
|
||||
return
|
||||
await route.continue_()
|
||||
except Exception:
|
||||
# Never let a routing error kill the page load
|
||||
try:
|
||||
await route.continue_()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _ensure_context(self) -> BrowserContext:
|
||||
"""Return a persistent context with a single coherent Chrome identity.
|
||||
|
||||
The context is reused across every date in a session so the browser
|
||||
cache stays warm (first page ~5 MB, each date after ~0.3 MB). It is
|
||||
warmed up on the homepage exactly once, here at creation time."""
|
||||
if self._context is not None:
|
||||
return self._context
|
||||
|
||||
browser = await self._ensure_browser()
|
||||
# Small viewport jitter so every session isn't pixel-identical
|
||||
width = 1920 + random.randint(-40, 40)
|
||||
|
|
@ -103,6 +230,16 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
timezone_id='Europe/London',
|
||||
extra_http_headers=self.CLIENT_HINT_HEADERS,
|
||||
)
|
||||
await context.route("**/*", self._route_handler)
|
||||
|
||||
# Warm up once per context (per IP) — subsequent date searches reuse
|
||||
# these cookies and cache rather than re-fetching the homepage.
|
||||
page = await context.new_page()
|
||||
await self._prepare_page(page)
|
||||
await self._warm_up(page)
|
||||
await page.close()
|
||||
|
||||
self._context = context
|
||||
return context
|
||||
|
||||
async def _prepare_page(self, page: Page):
|
||||
|
|
@ -368,7 +505,9 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
pages: int = 2
|
||||
) -> ScraperResult:
|
||||
"""
|
||||
Scrape booking.com location search results.
|
||||
Scrape booking.com location search results, rotating the proxy IP if
|
||||
a session looks soft-blocked (challenge page, or page 1 rendered
|
||||
nothing). With no proxy configured this is a single attempt.
|
||||
|
||||
Args:
|
||||
location: Location name
|
||||
|
|
@ -380,26 +519,55 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
Returns:
|
||||
ScraperResult with hotels and rates found
|
||||
"""
|
||||
max_attempts = 3 if self._proxy_enabled() else 1
|
||||
result = None
|
||||
|
||||
for attempt in range(max_attempts):
|
||||
result = await self._scrape_once(location, check_in, check_out, adults, pages)
|
||||
|
||||
# Soft-block signals: an explicit challenge, or a "successful" load
|
||||
# that yielded zero hotels (page 1 never rendered results).
|
||||
soft_blocked = result.blocked or (result.success and not result.hotels)
|
||||
if not soft_blocked:
|
||||
return result
|
||||
|
||||
if attempt < max_attempts - 1:
|
||||
logger.warning(
|
||||
f"Attempt {attempt + 1}/{max_attempts} for {check_in} looked blocked "
|
||||
f"(blocked={result.blocked}, hotels={len(result.hotels)}) — rotating IP"
|
||||
)
|
||||
await self.rotate_session()
|
||||
await asyncio.sleep(random.uniform(1.0, 2.0))
|
||||
|
||||
return result
|
||||
|
||||
async def _scrape_once(
|
||||
self,
|
||||
location: str,
|
||||
check_in: date,
|
||||
check_out: date,
|
||||
adults: int,
|
||||
pages: int
|
||||
) -> ScraperResult:
|
||||
"""A single scrape attempt for one date on the current IP/session."""
|
||||
all_hotels = []
|
||||
all_rates = []
|
||||
seen_hotel_ids = set()
|
||||
pages_ok = 0
|
||||
|
||||
context = None
|
||||
page = None
|
||||
|
||||
try:
|
||||
context = await self._create_context()
|
||||
# Persistent, already-warmed context (cache stays hot across dates)
|
||||
context = await self._ensure_context()
|
||||
page = await context.new_page()
|
||||
await self._prepare_page(page)
|
||||
|
||||
# Land on the homepage first so the search carries session cookies
|
||||
await self._warm_up(page)
|
||||
|
||||
for page_num in range(pages):
|
||||
# Random delay between pages (3-7 seconds)
|
||||
# Short jittered delay between pages. Kept modest because with
|
||||
# rotate-on-block a burned IP costs nothing — throughput wins.
|
||||
if page_num > 0:
|
||||
delay = random.uniform(3, 7)
|
||||
delay = random.uniform(1.5, 3.5)
|
||||
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
|
|
@ -488,10 +656,10 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
pages_ok=pages_ok,
|
||||
)
|
||||
finally:
|
||||
# Close only the page — the context persists across dates so its
|
||||
# cache stays warm. rotate_session()/close() dispose of the context.
|
||||
if page:
|
||||
await page.close()
|
||||
if context:
|
||||
await context.close()
|
||||
|
||||
async def scrape_hotel_page(
|
||||
self,
|
||||
|
|
@ -515,6 +683,12 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
|
||||
async def close(self):
|
||||
"""Clean up browser resources."""
|
||||
if self._context:
|
||||
try:
|
||||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._context = None
|
||||
if self._browser:
|
||||
await self._browser.close()
|
||||
self._browser = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue