Block images via Chrome flag, not route interception; restore DataImpulse URL auth
context.route() (request interception) breaks URL-embedded proxy auth in Chromium — DataImpulse proactive CONNECT auth stops working when interception is active. Replace route handler with --blink-settings=imagesEnabled=false Chrome flag to block images without touching the interception layer. Restore DataImpulse URL-embedded credentials (unchanged from before IPRoyal work). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d223a449c3
commit
d28c738475
2 changed files with 14 additions and 37 deletions
|
|
@ -109,9 +109,16 @@ def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[di
|
||||||
return None
|
return None
|
||||||
user, pwd = _build_auth(cfg, session_id)
|
user, pwd = _build_auth(cfg, session_id)
|
||||||
server = f"http://{cfg['host']}:{cfg['port']}"
|
server = f"http://{cfg['host']}:{cfg['port']}"
|
||||||
# Both providers use separate fields — URL-embedded credentials stop being
|
if _provider(cfg) == 'iproyal':
|
||||||
# forwarded by Chromium when request interception (context.route) is active.
|
# IPRoyal requires separate fields — Chromium silently drops
|
||||||
return {'server': server, 'username': user, 'password': pwd}
|
# URL-embedded credentials for this provider.
|
||||||
|
return {'server': server, 'username': user, 'password': pwd}
|
||||||
|
from urllib.parse import quote
|
||||||
|
# DataImpulse: credentials in URL so Chromium sends Proxy-Authorization
|
||||||
|
# proactively on CONNECT (avoids a ~14s 407 round-trip).
|
||||||
|
return {
|
||||||
|
'server': f"http://{quote(user, safe='')}:{quote(pwd, safe='')}@{cfg['host']}:{cfg['port']}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def httpx_proxy(cfg: dict, session_id: Optional[str] = None):
|
def httpx_proxy(cfg: dict, session_id: Optional[str] = None):
|
||||||
|
|
|
||||||
|
|
@ -55,39 +55,6 @@ _VIEWPORTS = [
|
||||||
{'width': 1600, 'height': 900},
|
{'width': 1600, 'height': 900},
|
||||||
]
|
]
|
||||||
|
|
||||||
# Resource types to abort — we only need the DOM and JS execution.
|
|
||||||
# Images, stylesheets, and fonts are served from cf.bstatic.com and account
|
|
||||||
# for ~75% of bandwidth; aborting them saves ~1.5 GB per scrape run.
|
|
||||||
_BLOCK_RESOURCE_TYPES = {'image', 'stylesheet', 'font', 'media'}
|
|
||||||
|
|
||||||
# Pure tracking/analytics domains with no effect on DOM or JS prices rendering.
|
|
||||||
_BLOCK_DOMAINS = {
|
|
||||||
'cdn.cookielaw.org',
|
|
||||||
'geolocation.onetrust.com',
|
|
||||||
'graph.facebook.com',
|
|
||||||
'platform-lookaside.fbsbx.com',
|
|
||||||
'lh3.googleusercontent.com',
|
|
||||||
'maps.googleapis.com',
|
|
||||||
'web-perf.booking.com',
|
|
||||||
'sink.gw.booking.com',
|
|
||||||
'otel-gw.booking.com',
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def _route_handler(route, request):
|
|
||||||
if request.resource_type in _BLOCK_RESOURCE_TYPES:
|
|
||||||
await route.abort()
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
host = request.url.split('/')[2].split(':')[0]
|
|
||||||
except IndexError:
|
|
||||||
host = ''
|
|
||||||
if host in _BLOCK_DOMAINS:
|
|
||||||
await route.abort()
|
|
||||||
return
|
|
||||||
await route.continue_()
|
|
||||||
|
|
||||||
|
|
||||||
# Injected into every new context to mask headless/webdriver signals.
|
# Injected into every new context to mask headless/webdriver signals.
|
||||||
_STEALTH_SCRIPT = """
|
_STEALTH_SCRIPT = """
|
||||||
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
||||||
|
|
@ -261,6 +228,10 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
||||||
'--disable-blink-features=AutomationControlled',
|
'--disable-blink-features=AutomationControlled',
|
||||||
'--disable-infobars',
|
'--disable-infobars',
|
||||||
'--disable-extensions',
|
'--disable-extensions',
|
||||||
|
# Block image loading at the engine level — avoids ~75% of
|
||||||
|
# proxy bandwidth (cf.bstatic.com CDN) without enabling
|
||||||
|
# request interception, which breaks proxy auth in Chromium.
|
||||||
|
'--blink-settings=imagesEnabled=false',
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -283,7 +254,6 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
||||||
**self._proxy_kwargs(session_id),
|
**self._proxy_kwargs(session_id),
|
||||||
)
|
)
|
||||||
await self._context.add_init_script(_STEALTH_SCRIPT)
|
await self._context.add_init_script(_STEALTH_SCRIPT)
|
||||||
await self._context.route('**/*', _route_handler)
|
|
||||||
self._requests_on_context = 0
|
self._requests_on_context = 0
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}"
|
f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue