Scraper stealth: headful+xvfb, playwright-stealth, coherent UA, warm-up, click-paginate
Tier 1+2 anti-detection to stop Booking.com throttling page 2: - Run Chromium headful under xvfb (Dockerfile) — headless leaks SwiftShader WebGL, empty plugins, missing chrome.runtime - playwright-stealth patches navigator.webdriver/plugins/WebGL vendor - Single coherent Chrome-121 identity: UA + matching sec-ch-ua client hints + platform (dropped the Firefox/Safari UA strings — a mismatched UA on a Chromium engine is a stronger tell than no rotation) - Homepage warm-up so search requests carry real session cookies + cookie consent dismiss - Paginate by clicking next (offset= deep-link was the page-2 tell), offset URL kept as fallback - Viewport jitter, mouse movement, longer scroll, selector retry on lazy load Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
b712196262
commit
349c795708
3 changed files with 148 additions and 36 deletions
|
|
@ -16,6 +16,11 @@ from urllib.parse import urlencode
|
|||
|
||||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
||||
|
||||
try:
|
||||
from playwright_stealth import stealth_async
|
||||
except ImportError: # pragma: no cover - stealth is optional at runtime
|
||||
stealth_async = None
|
||||
|
||||
from .base import (
|
||||
ScraperBackend,
|
||||
ScraperResult,
|
||||
|
|
@ -31,20 +36,31 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
"""
|
||||
Local Playwright backend using Chromium.
|
||||
|
||||
Features:
|
||||
- Rotates user agents
|
||||
- Random delays between requests
|
||||
- Mimics human scroll behavior
|
||||
- Uses data-testid selectors for stability
|
||||
Stealth measures:
|
||||
- Runs headful (via xvfb in the container) — headless leaks SwiftShader
|
||||
WebGL, empty plugins, missing chrome.runtime
|
||||
- playwright-stealth patches navigator.webdriver, plugins, WebGL vendor
|
||||
- ONE consistent modern-Chrome identity: UA + matching sec-ch-ua client
|
||||
hints + platform (a mismatched UA is worse than none)
|
||||
- Warms up via the homepage so search requests carry real session cookies
|
||||
- Paginates by clicking "next" rather than deep-linking ?offset=25
|
||||
- Random delays, viewport jitter, mouse movement, human-like scroll
|
||||
"""
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
|
||||
]
|
||||
# A single coherent identity. The UA, the sec-ch-ua hints and the
|
||||
# platform must all agree or the mismatch itself is a bot signal.
|
||||
CHROME_VERSION = "121"
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
|
||||
)
|
||||
CLIENT_HINT_HEADERS = {
|
||||
"sec-ch-ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Windows"',
|
||||
"accept-language": "en-GB,en;q=0.9",
|
||||
}
|
||||
HOMEPAGE = "https://www.booking.com/index.en-gb.html"
|
||||
|
||||
def __init__(self, proxy_config: dict = None):
|
||||
"""
|
||||
|
|
@ -62,26 +78,61 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
if self._browser is None or not self._browser.is_connected():
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=True,
|
||||
# Headful under xvfb — far cleaner fingerprint than headless.
|
||||
headless=False,
|
||||
args=[
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--no-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-features=IsolateOrigins,site-per-process',
|
||||
'--start-maximized',
|
||||
]
|
||||
)
|
||||
return self._browser
|
||||
|
||||
async def _create_context(self) -> BrowserContext:
|
||||
"""Create a new browser context with random user agent."""
|
||||
"""Create a browser context with a single coherent Chrome identity."""
|
||||
browser = await self._ensure_browser()
|
||||
# Small viewport jitter so every session isn't pixel-identical
|
||||
width = 1920 + random.randint(-40, 40)
|
||||
height = 1080 + random.randint(-30, 30)
|
||||
context = await browser.new_context(
|
||||
user_agent=random.choice(self.USER_AGENTS),
|
||||
viewport={'width': 1920, 'height': 1080},
|
||||
user_agent=self.USER_AGENT,
|
||||
viewport={'width': width, 'height': height},
|
||||
locale='en-GB',
|
||||
timezone_id='Europe/London',
|
||||
extra_http_headers=self.CLIENT_HINT_HEADERS,
|
||||
)
|
||||
return context
|
||||
|
||||
async def _prepare_page(self, page: Page):
|
||||
"""Apply stealth patches to a freshly created page."""
|
||||
if stealth_async is not None:
|
||||
try:
|
||||
await stealth_async(page)
|
||||
except Exception as e:
|
||||
logger.warning(f"stealth_async failed, continuing without: {e}")
|
||||
|
||||
async def _warm_up(self, page: Page):
|
||||
"""Visit the homepage first so search requests carry session cookies."""
|
||||
try:
|
||||
await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=30000)
|
||||
await asyncio.sleep(random.uniform(1.5, 3.0))
|
||||
# Dismiss the cookie-consent dialog if present
|
||||
for sel in ['#onetrust-accept-btn-handler',
|
||||
'[aria-label="Accept"]',
|
||||
'[data-testid="cookie-banner-accept"]']:
|
||||
try:
|
||||
btn = await page.query_selector(sel)
|
||||
if btn:
|
||||
await btn.click()
|
||||
await asyncio.sleep(random.uniform(0.4, 0.9))
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning(f"Homepage warm-up failed, continuing: {e}")
|
||||
|
||||
def _build_search_url(
|
||||
self,
|
||||
location: str,
|
||||
|
|
@ -137,21 +188,61 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
async def _human_like_scroll(self, page: Page):
|
||||
"""Simulate human-like scrolling behavior."""
|
||||
# Scroll down in increments
|
||||
for _ in range(3):
|
||||
for _ in range(random.randint(3, 6)):
|
||||
await page.mouse.wheel(0, random.randint(300, 600))
|
||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||
|
||||
async def _human_like_mouse(self, page: Page):
|
||||
"""A few random mouse moves — real users generate pointer events."""
|
||||
try:
|
||||
for _ in range(random.randint(2, 4)):
|
||||
await page.mouse.move(random.randint(100, 1400), random.randint(150, 800))
|
||||
await asyncio.sleep(random.uniform(0.1, 0.4))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _go_to_next_page(self, page: Page) -> bool:
|
||||
"""Click the pagination 'next' control. Returns True if navigation happened."""
|
||||
selectors = [
|
||||
'[data-testid="pagination-next-btn"]',
|
||||
'button[aria-label="Next page"]',
|
||||
'a[aria-label="Next page"]',
|
||||
]
|
||||
for sel in selectors:
|
||||
try:
|
||||
btn = await page.query_selector(sel)
|
||||
if btn and await btn.is_enabled():
|
||||
await btn.scroll_into_view_if_needed()
|
||||
await asyncio.sleep(random.uniform(0.3, 0.7))
|
||||
await btn.click()
|
||||
# Results re-render in place; wait for network to settle
|
||||
try:
|
||||
await page.wait_for_load_state('networkidle', timeout=15000)
|
||||
except Exception:
|
||||
await asyncio.sleep(2)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Next-page selector {sel} failed: {e}")
|
||||
continue
|
||||
return False
|
||||
|
||||
async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
|
||||
"""Extract hotel and rate data from search results page."""
|
||||
hotels = []
|
||||
rates = []
|
||||
|
||||
# Wait for property cards - booking.com uses data-testid
|
||||
# Wait for property cards - booking.com uses data-testid. Retry once
|
||||
# with a scroll nudge before giving up: cards can lazy-load late.
|
||||
try:
|
||||
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
|
||||
except Exception as e:
|
||||
logger.warning(f"No property cards found: {e}")
|
||||
return hotels, rates
|
||||
logger.warning(f"No property cards on first wait, retrying: {e}")
|
||||
try:
|
||||
await self._human_like_scroll(page)
|
||||
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
|
||||
except Exception as e2:
|
||||
logger.warning(f"No property cards found after retry: {e2}")
|
||||
return hotels, rates
|
||||
|
||||
# Get all property cards
|
||||
cards = await page.query_selector_all('[data-testid="property-card"]')
|
||||
|
|
@ -300,6 +391,10 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
try:
|
||||
context = await self._create_context()
|
||||
page = await context.new_page()
|
||||
await self._prepare_page(page)
|
||||
|
||||
# Land on the homepage first so the search carries session cookies
|
||||
await self._warm_up(page)
|
||||
|
||||
for page_num in range(pages):
|
||||
# Random delay between pages (3-7 seconds)
|
||||
|
|
@ -308,20 +403,31 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
# Build URL with offset for pagination (25 results per page)
|
||||
url = self._build_search_url(
|
||||
location, check_in, check_out, adults,
|
||||
offset=page_num * 25
|
||||
)
|
||||
|
||||
logger.info(f"Scraping page {page_num + 1}: {url}")
|
||||
|
||||
page_loaded = True
|
||||
try:
|
||||
await page.goto(url, wait_until='networkidle', timeout=30000)
|
||||
except Exception as e:
|
||||
logger.warning(f"Page load timeout, continuing: {e}")
|
||||
page_loaded = False
|
||||
if page_num == 0:
|
||||
# Navigate to page 1 by URL
|
||||
url = self._build_search_url(location, check_in, check_out, adults)
|
||||
logger.info(f"Scraping page {page_num + 1}: {url}")
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
except Exception as e:
|
||||
logger.warning(f"Page load timeout, continuing: {e}")
|
||||
page_loaded = False
|
||||
else:
|
||||
# Paginate by clicking "next" like a human — deep-linking
|
||||
# ?offset=25 is a stronger bot signal and gets throttled
|
||||
clicked = await self._go_to_next_page(page)
|
||||
if not clicked:
|
||||
# Fall back to offset URL if the control isn't found
|
||||
url = self._build_search_url(
|
||||
location, check_in, check_out, adults, offset=page_num * 25
|
||||
)
|
||||
logger.info(f"Next-button not found, offset fallback: {url}")
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
except Exception as e:
|
||||
logger.warning(f"Page load timeout, continuing: {e}")
|
||||
page_loaded = False
|
||||
|
||||
# Check for blocking
|
||||
content = await page.content()
|
||||
|
|
@ -339,10 +445,11 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
pages_ok=pages_ok,
|
||||
)
|
||||
|
||||
# Human-like scrolling
|
||||
# Human-like mouse movement + scrolling
|
||||
await self._human_like_mouse(page)
|
||||
await self._human_like_scroll(page)
|
||||
|
||||
# Extract data
|
||||
# Extract data (retries the selector once on timeout)
|
||||
hotels, rates = await self._extract_search_results(page, check_in)
|
||||
|
||||
# A page counts as clean if it loaded fully and parsed.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue