diff --git a/backend/services/proxy.py b/backend/services/proxy.py index 0874341..b179218 100644 --- a/backend/services/proxy.py +++ b/backend/services/proxy.py @@ -109,9 +109,16 @@ def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[di return None user, pwd = _build_auth(cfg, session_id) server = f"http://{cfg['host']}:{cfg['port']}" - # Both providers use separate fields — URL-embedded credentials stop being - # forwarded by Chromium when request interception (context.route) is active. - return {'server': server, 'username': user, 'password': pwd} + if _provider(cfg) == 'iproyal': + # IPRoyal requires separate fields — Chromium silently drops + # URL-embedded credentials for this provider. + return {'server': server, 'username': user, 'password': pwd} + from urllib.parse import quote + # DataImpulse: credentials in URL so Chromium sends Proxy-Authorization + # proactively on CONNECT (avoids a ~14s 407 round-trip). + return { + 'server': f"http://{quote(user, safe='')}:{quote(pwd, safe='')}@{cfg['host']}:{cfg['port']}", + } def httpx_proxy(cfg: dict, session_id: Optional[str] = None): diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py index 6cd649c..81f27e4 100644 --- a/backend/services/scraper_backends/playwright_hotel_page.py +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -55,39 +55,6 @@ _VIEWPORTS = [ {'width': 1600, 'height': 900}, ] -# Resource types to abort — we only need the DOM and JS execution. -# Images, stylesheets, and fonts are served from cf.bstatic.com and account -# for ~75% of bandwidth; aborting them saves ~1.5 GB per scrape run. -_BLOCK_RESOURCE_TYPES = {'image', 'stylesheet', 'font', 'media'} - -# Pure tracking/analytics domains with no effect on DOM or JS prices rendering. -_BLOCK_DOMAINS = { - 'cdn.cookielaw.org', - 'geolocation.onetrust.com', - 'graph.facebook.com', - 'platform-lookaside.fbsbx.com', - 'lh3.googleusercontent.com', - 'maps.googleapis.com', - 'web-perf.booking.com', - 'sink.gw.booking.com', - 'otel-gw.booking.com', -} - - -async def _route_handler(route, request): - if request.resource_type in _BLOCK_RESOURCE_TYPES: - await route.abort() - return - try: - host = request.url.split('/')[2].split(':')[0] - except IndexError: - host = '' - if host in _BLOCK_DOMAINS: - await route.abort() - return - await route.continue_() - - # Injected into every new context to mask headless/webdriver signals. _STEALTH_SCRIPT = """ Object.defineProperty(navigator, 'webdriver', {get: () => undefined}); @@ -261,6 +228,10 @@ class PlaywrightHotelPageBackend(ScraperBackend): '--disable-blink-features=AutomationControlled', '--disable-infobars', '--disable-extensions', + # Block image loading at the engine level — avoids ~75% of + # proxy bandwidth (cf.bstatic.com CDN) without enabling + # request interception, which breaks proxy auth in Chromium. + '--blink-settings=imagesEnabled=false', ], ) @@ -283,7 +254,6 @@ class PlaywrightHotelPageBackend(ScraperBackend): **self._proxy_kwargs(session_id), ) await self._context.add_init_script(_STEALTH_SCRIPT) - await self._context.route('**/*', _route_handler) self._requests_on_context = 0 logger.debug( f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}"