From ca76bc2f9067b018ebd53f5ab425d87f94f15a4b Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 10 Jul 2026 00:21:42 +0000 Subject: [PATCH] Fix proxy never being used in hotel-page scraper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _proxy_enabled() was checking cfg.get('server') but load_config() returns {host, port, username, ...} — no 'server' key. Server URL is built by proxy_util.playwright_proxy(). This meant proxy was silently disabled and _proxy_kwargs() would also KeyError if called. - _proxy_enabled() now delegates to proxy_util.is_enabled() (checks host+username) - _proxy_kwargs() now calls proxy_util.playwright_proxy() with credentials embedded in the URL (avoids 14s DataImpulse 407 round-trip) - _get_context() generates a sticky session_id per context so each worker gets a distinct residential IP lane Co-Authored-By: Claude Sonnet 4.6 --- .../scraper_backends/playwright_hotel_page.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py index 0814299..1c94247 100644 --- a/backend/services/scraper_backends/playwright_hotel_page.py +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -21,6 +21,7 @@ Rate plan variants per room type (typical): import asyncio import logging import re +import uuid from datetime import date from decimal import Decimal from typing import List, Optional, Tuple @@ -29,6 +30,7 @@ from urllib.parse import urlparse from playwright.async_api import async_playwright, Browser, BrowserContext, Page from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus +from services import proxy as proxy_util logger = logging.getLogger(__name__) @@ -161,18 +163,13 @@ class PlaywrightHotelPageBackend(ScraperBackend): self._max_requests_per_context = 40 def _proxy_enabled(self) -> bool: - return bool(self._proxy_config.get('server')) + return proxy_util.is_enabled(self._proxy_config) - def _proxy_kwargs(self) -> dict: - if not self._proxy_enabled(): - return {} - cfg = self._proxy_config - proxy = {'server': cfg['server']} - if cfg.get('username'): - proxy['username'] = cfg['username'] - if cfg.get('password'): - proxy['password'] = cfg['password'] - return {'proxy': proxy} + def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict: + # Credentials embedded in URL — separate username/password fields cause + # a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment). + proxy = proxy_util.playwright_proxy(self._proxy_config, session_id) + return {'proxy': proxy} if proxy else {} async def _start(self): if not self._pw: @@ -191,16 +188,17 @@ class PlaywrightHotelPageBackend(ScraperBackend): await self._context.close() except Exception: pass + session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None self._context = await self._browser.new_context( user_agent=( 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/125.0.0.0 Safari/537.36' ), - **self._proxy_kwargs(), + **self._proxy_kwargs(session_id), ) self._requests_on_context = 0 - logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else "")) + logger.debug("Opened new browser context" + (f" (proxy session {session_id})" if session_id else "")) return self._context async def _rotate_context(self):