hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Increase Playwright page.goto timeout from 30s to 90s

Browse source 
DataImpulse residential proxy takes ~29s to route through Booking.com's
Cloudflare edge — the previous 30s timeout was too short and the page
never had a chance to deliver its body. Also raised wait_for_selector
from 15s to 30s so property cards have time to render post-load.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 17:23:41 +00:00
parent 617ff932bb
commit e53c482141
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
backend/services/scraper_backends/playwright_local.py
View file

@ -247,7 +247,7 @@ class PlaywrightLocalBackend(ScraperBackend):
async def _warm_up(self, page: Page):
"""Visit the homepage first so search requests carry session cookies."""
try:
await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=30000)
await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=90000)
await asyncio.sleep(random.uniform(1.5, 3.0))
# Dismiss the cookie-consent dialog if present
for sel in ['#onetrust-accept-btn-handler',
@ -379,12 +379,12 @@ class PlaywrightLocalBackend(ScraperBackend):
# Wait for property cards - booking.com uses data-testid. Retry once
# with a scroll nudge before giving up: cards can lazy-load late.
try:
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
await page.wait_for_selector('[data-testid="property-card"]', timeout=30000)
except Exception as e:
logger.warning(f"No property cards on first wait, retrying: {e}")
try:
await self._human_like_scroll(page)
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
await page.wait_for_selector('[data-testid="property-card"]', timeout=30000)
except Exception as e2:
logger.warning(f"No property cards found after retry: {e2}")
return hotels, rates
@ -596,7 +596,7 @@ class PlaywrightLocalBackend(ScraperBackend):
dest_id=dest_id, location_params=location_params)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
await page.goto(url, wait_until='domcontentloaded', timeout=90000)
except Exception as e:
logger.warning(f"Page load timeout, continuing: {e}")
page_loaded = False