Fix proxy never being used in hotel-page scraper
_proxy_enabled() was checking cfg.get('server') but load_config() returns
{host, port, username, ...} — no 'server' key. Server URL is built by
proxy_util.playwright_proxy(). This meant proxy was silently disabled
and _proxy_kwargs() would also KeyError if called.
- _proxy_enabled() now delegates to proxy_util.is_enabled() (checks host+username)
- _proxy_kwargs() now calls proxy_util.playwright_proxy() with credentials
embedded in the URL (avoids 14s DataImpulse 407 round-trip)
- _get_context() generates a sticky session_id per context so each worker
gets a distinct residential IP lane
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
1b985e5a16
commit
ca76bc2f90
1 changed files with 11 additions and 13 deletions
|
|
@ -21,6 +21,7 @@ Rate plan variants per room type (typical):
|
|||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from typing import List, Optional, Tuple
|
||||
|
|
@ -29,6 +30,7 @@ from urllib.parse import urlparse
|
|||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
||||
|
||||
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
|
||||
from services import proxy as proxy_util
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -161,18 +163,13 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
|||
self._max_requests_per_context = 40
|
||||
|
||||
def _proxy_enabled(self) -> bool:
|
||||
return bool(self._proxy_config.get('server'))
|
||||
return proxy_util.is_enabled(self._proxy_config)
|
||||
|
||||
def _proxy_kwargs(self) -> dict:
|
||||
if not self._proxy_enabled():
|
||||
return {}
|
||||
cfg = self._proxy_config
|
||||
proxy = {'server': cfg['server']}
|
||||
if cfg.get('username'):
|
||||
proxy['username'] = cfg['username']
|
||||
if cfg.get('password'):
|
||||
proxy['password'] = cfg['password']
|
||||
return {'proxy': proxy}
|
||||
def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict:
|
||||
# Credentials embedded in URL — separate username/password fields cause
|
||||
# a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment).
|
||||
proxy = proxy_util.playwright_proxy(self._proxy_config, session_id)
|
||||
return {'proxy': proxy} if proxy else {}
|
||||
|
||||
async def _start(self):
|
||||
if not self._pw:
|
||||
|
|
@ -191,16 +188,17 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
|||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None
|
||||
self._context = await self._browser.new_context(
|
||||
user_agent=(
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
'Chrome/125.0.0.0 Safari/537.36'
|
||||
),
|
||||
**self._proxy_kwargs(),
|
||||
**self._proxy_kwargs(session_id),
|
||||
)
|
||||
self._requests_on_context = 0
|
||||
logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else ""))
|
||||
logger.debug("Opened new browser context" + (f" (proxy session {session_id})" if session_id else ""))
|
||||
return self._context
|
||||
|
||||
async def _rotate_context(self):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue