From e53c482141aaa85c612a2c52696a271a4cc4dd7a Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 17:23:41 +0000 Subject: [PATCH] Increase Playwright page.goto timeout from 30s to 90s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DataImpulse residential proxy takes ~29s to route through Booking.com's Cloudflare edge — the previous 30s timeout was too short and the page never had a chance to deliver its body. Also raised wait_for_selector from 15s to 30s so property cards have time to render post-load. Co-Authored-By: Claude Sonnet 4.6 --- backend/services/scraper_backends/playwright_local.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 2c9026c..cb54fc7 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -247,7 +247,7 @@ class PlaywrightLocalBackend(ScraperBackend): async def _warm_up(self, page: Page): """Visit the homepage first so search requests carry session cookies.""" try: - await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=30000) + await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=90000) await asyncio.sleep(random.uniform(1.5, 3.0)) # Dismiss the cookie-consent dialog if present for sel in ['#onetrust-accept-btn-handler', @@ -379,12 +379,12 @@ class PlaywrightLocalBackend(ScraperBackend): # Wait for property cards - booking.com uses data-testid. Retry once # with a scroll nudge before giving up: cards can lazy-load late. try: - await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) + await page.wait_for_selector('[data-testid="property-card"]', timeout=30000) except Exception as e: logger.warning(f"No property cards on first wait, retrying: {e}") try: await self._human_like_scroll(page) - await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) + await page.wait_for_selector('[data-testid="property-card"]', timeout=30000) except Exception as e2: logger.warning(f"No property cards found after retry: {e2}") return hotels, rates @@ -596,7 +596,7 @@ class PlaywrightLocalBackend(ScraperBackend): dest_id=dest_id, location_params=location_params) logger.info(f"Scraping page {page_num + 1}: {url}") try: - await page.goto(url, wait_until='domcontentloaded', timeout=30000) + await page.goto(url, wait_until='domcontentloaded', timeout=90000) except Exception as e: logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False