Fix proxy never being used in hotel-page scraper
_proxy_enabled() was checking cfg.get('server') but load_config() returns
{host, port, username, ...} — no 'server' key. Server URL is built by
proxy_util.playwright_proxy(). This meant proxy was silently disabled
and _proxy_kwargs() would also KeyError if called.
- _proxy_enabled() now delegates to proxy_util.is_enabled() (checks host+username)
- _proxy_kwargs() now calls proxy_util.playwright_proxy() with credentials
embedded in the URL (avoids 14s DataImpulse 407 round-trip)
- _get_context() generates a sticky session_id per context so each worker
gets a distinct residential IP lane
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
1b985e5a16
commit
ca76bc2f90
1 changed files with 11 additions and 13 deletions
|
|
@ -21,6 +21,7 @@ Rate plan variants per room type (typical):
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
import uuid
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from typing import List, Optional, Tuple
|
from typing import List, Optional, Tuple
|
||||||
|
|
@ -29,6 +30,7 @@ from urllib.parse import urlparse
|
||||||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
||||||
|
|
||||||
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
|
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
|
||||||
|
from services import proxy as proxy_util
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -161,18 +163,13 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
||||||
self._max_requests_per_context = 40
|
self._max_requests_per_context = 40
|
||||||
|
|
||||||
def _proxy_enabled(self) -> bool:
|
def _proxy_enabled(self) -> bool:
|
||||||
return bool(self._proxy_config.get('server'))
|
return proxy_util.is_enabled(self._proxy_config)
|
||||||
|
|
||||||
def _proxy_kwargs(self) -> dict:
|
def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict:
|
||||||
if not self._proxy_enabled():
|
# Credentials embedded in URL — separate username/password fields cause
|
||||||
return {}
|
# a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment).
|
||||||
cfg = self._proxy_config
|
proxy = proxy_util.playwright_proxy(self._proxy_config, session_id)
|
||||||
proxy = {'server': cfg['server']}
|
return {'proxy': proxy} if proxy else {}
|
||||||
if cfg.get('username'):
|
|
||||||
proxy['username'] = cfg['username']
|
|
||||||
if cfg.get('password'):
|
|
||||||
proxy['password'] = cfg['password']
|
|
||||||
return {'proxy': proxy}
|
|
||||||
|
|
||||||
async def _start(self):
|
async def _start(self):
|
||||||
if not self._pw:
|
if not self._pw:
|
||||||
|
|
@ -191,16 +188,17 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
||||||
await self._context.close()
|
await self._context.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None
|
||||||
self._context = await self._browser.new_context(
|
self._context = await self._browser.new_context(
|
||||||
user_agent=(
|
user_agent=(
|
||||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||||
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||||
'Chrome/125.0.0.0 Safari/537.36'
|
'Chrome/125.0.0.0 Safari/537.36'
|
||||||
),
|
),
|
||||||
**self._proxy_kwargs(),
|
**self._proxy_kwargs(session_id),
|
||||||
)
|
)
|
||||||
self._requests_on_context = 0
|
self._requests_on_context = 0
|
||||||
logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else ""))
|
logger.debug("Opened new browser context" + (f" (proxy session {session_id})" if session_id else ""))
|
||||||
return self._context
|
return self._context
|
||||||
|
|
||||||
async def _rotate_context(self):
|
async def _rotate_context(self):
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue