Block images/CSS/fonts in Playwright to cut ~75% bandwidth
cf.bstatic.com (images + CSS CDN) was consuming 1.5 GB per scrape run. Added route interception to abort image, stylesheet, font, and media resources, plus pure tracking domains. The DOM extraction only reads HTML attributes — no visual resources are needed. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d72f14ae49
commit
418e0d2fcc
1 changed files with 34 additions and 0 deletions
|
|
@ -55,6 +55,39 @@ _VIEWPORTS = [
|
||||||
{'width': 1600, 'height': 900},
|
{'width': 1600, 'height': 900},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Resource types to abort — we only need the DOM and JS execution.
|
||||||
|
# Images, stylesheets, and fonts are served from cf.bstatic.com and account
|
||||||
|
# for ~75% of bandwidth; aborting them saves ~1.5 GB per scrape run.
|
||||||
|
_BLOCK_RESOURCE_TYPES = {'image', 'stylesheet', 'font', 'media'}
|
||||||
|
|
||||||
|
# Pure tracking/analytics domains with no effect on DOM or JS prices rendering.
|
||||||
|
_BLOCK_DOMAINS = {
|
||||||
|
'cdn.cookielaw.org',
|
||||||
|
'geolocation.onetrust.com',
|
||||||
|
'graph.facebook.com',
|
||||||
|
'platform-lookaside.fbsbx.com',
|
||||||
|
'lh3.googleusercontent.com',
|
||||||
|
'maps.googleapis.com',
|
||||||
|
'web-perf.booking.com',
|
||||||
|
'sink.gw.booking.com',
|
||||||
|
'otel-gw.booking.com',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _route_handler(route, request):
|
||||||
|
if request.resource_type in _BLOCK_RESOURCE_TYPES:
|
||||||
|
await route.abort()
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
host = request.url.split('/')[2].split(':')[0]
|
||||||
|
except IndexError:
|
||||||
|
host = ''
|
||||||
|
if host in _BLOCK_DOMAINS:
|
||||||
|
await route.abort()
|
||||||
|
return
|
||||||
|
await route.continue_()
|
||||||
|
|
||||||
|
|
||||||
# Injected into every new context to mask headless/webdriver signals.
|
# Injected into every new context to mask headless/webdriver signals.
|
||||||
_STEALTH_SCRIPT = """
|
_STEALTH_SCRIPT = """
|
||||||
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
||||||
|
|
@ -250,6 +283,7 @@ class PlaywrightHotelPageBackend(ScraperBackend):
|
||||||
**self._proxy_kwargs(session_id),
|
**self._proxy_kwargs(session_id),
|
||||||
)
|
)
|
||||||
await self._context.add_init_script(_STEALTH_SCRIPT)
|
await self._context.add_init_script(_STEALTH_SCRIPT)
|
||||||
|
await self._context.route('**/*', _route_handler)
|
||||||
self._requests_on_context = 0
|
self._requests_on_context = 0
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}"
|
f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue