Fix proxy never being used in hotel-page scraper

_proxy_enabled() was checking cfg.get('server') but load_config() returns
{host, port, username, ...} — no 'server' key. Server URL is built by
proxy_util.playwright_proxy(). This meant proxy was silently disabled
and _proxy_kwargs() would also KeyError if called.

- _proxy_enabled() now delegates to proxy_util.is_enabled() (checks host+username)
- _proxy_kwargs() now calls proxy_util.playwright_proxy() with credentials
  embedded in the URL (avoids 14s DataImpulse 407 round-trip)
- _get_context() generates a sticky session_id per context so each worker
  gets a distinct residential IP lane

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-10 00:21:42 +00:00
parent 1b985e5a16
commit ca76bc2f90

View file

@ -21,6 +21,7 @@ Rate plan variants per room type (typical):
import asyncio import asyncio
import logging import logging
import re import re
import uuid
from datetime import date from datetime import date
from decimal import Decimal from decimal import Decimal
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
@ -29,6 +30,7 @@ from urllib.parse import urlparse
from playwright.async_api import async_playwright, Browser, BrowserContext, Page from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
from services import proxy as proxy_util
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -161,18 +163,13 @@ class PlaywrightHotelPageBackend(ScraperBackend):
self._max_requests_per_context = 40 self._max_requests_per_context = 40
def _proxy_enabled(self) -> bool: def _proxy_enabled(self) -> bool:
return bool(self._proxy_config.get('server')) return proxy_util.is_enabled(self._proxy_config)
def _proxy_kwargs(self) -> dict: def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict:
if not self._proxy_enabled(): # Credentials embedded in URL — separate username/password fields cause
return {} # a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment).
cfg = self._proxy_config proxy = proxy_util.playwright_proxy(self._proxy_config, session_id)
proxy = {'server': cfg['server']} return {'proxy': proxy} if proxy else {}
if cfg.get('username'):
proxy['username'] = cfg['username']
if cfg.get('password'):
proxy['password'] = cfg['password']
return {'proxy': proxy}
async def _start(self): async def _start(self):
if not self._pw: if not self._pw:
@ -191,16 +188,17 @@ class PlaywrightHotelPageBackend(ScraperBackend):
await self._context.close() await self._context.close()
except Exception: except Exception:
pass pass
session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None
self._context = await self._browser.new_context( self._context = await self._browser.new_context(
user_agent=( user_agent=(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) ' 'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/125.0.0.0 Safari/537.36' 'Chrome/125.0.0.0 Safari/537.36'
), ),
**self._proxy_kwargs(), **self._proxy_kwargs(session_id),
) )
self._requests_on_context = 0 self._requests_on_context = 0
logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else "")) logger.debug("Opened new browser context" + (f" (proxy session {session_id})" if session_id else ""))
return self._context return self._context
async def _rotate_context(self): async def _rotate_context(self):