Fix proxy never being used in hotel-page scraper

_proxy_enabled() was checking cfg.get('server') but load_config() returns
{host, port, username, ...} — no 'server' key. Server URL is built by
proxy_util.playwright_proxy(). This meant proxy was silently disabled
and _proxy_kwargs() would also KeyError if called.

- _proxy_enabled() now delegates to proxy_util.is_enabled() (checks host+username)
- _proxy_kwargs() now calls proxy_util.playwright_proxy() with credentials
  embedded in the URL (avoids 14s DataImpulse 407 round-trip)
- _get_context() generates a sticky session_id per context so each worker
  gets a distinct residential IP lane

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-10 00:21:42 +00:00
parent 1b985e5a16
commit ca76bc2f90

View file

@ -21,6 +21,7 @@ Rate plan variants per room type (typical):
import asyncio
import logging
import re
import uuid
from datetime import date
from decimal import Decimal
from typing import List, Optional, Tuple
@ -29,6 +30,7 @@ from urllib.parse import urlparse
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
from services import proxy as proxy_util
logger = logging.getLogger(__name__)
@ -161,18 +163,13 @@ class PlaywrightHotelPageBackend(ScraperBackend):
self._max_requests_per_context = 40
def _proxy_enabled(self) -> bool:
return bool(self._proxy_config.get('server'))
return proxy_util.is_enabled(self._proxy_config)
def _proxy_kwargs(self) -> dict:
if not self._proxy_enabled():
return {}
cfg = self._proxy_config
proxy = {'server': cfg['server']}
if cfg.get('username'):
proxy['username'] = cfg['username']
if cfg.get('password'):
proxy['password'] = cfg['password']
return {'proxy': proxy}
def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict:
# Credentials embedded in URL — separate username/password fields cause
# a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment).
proxy = proxy_util.playwright_proxy(self._proxy_config, session_id)
return {'proxy': proxy} if proxy else {}
async def _start(self):
if not self._pw:
@ -191,16 +188,17 @@ class PlaywrightHotelPageBackend(ScraperBackend):
await self._context.close()
except Exception:
pass
session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None
self._context = await self._browser.new_context(
user_agent=(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/125.0.0.0 Safari/537.36'
),
**self._proxy_kwargs(),
**self._proxy_kwargs(session_id),
)
self._requests_on_context = 0
logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else ""))
logger.debug("Opened new browser context" + (f" (proxy session {session_id})" if session_id else ""))
return self._context
async def _rotate_context(self):