diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 660ec84..698f4c4 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -536,9 +536,10 @@ class PlaywrightLocalBackend(ScraperBackend): result = await self._scrape_once( location, check_in, check_out, adults, pages, dest_id, location_params) - # Soft-block signals: an explicit challenge, or a "successful" load - # that yielded zero hotels (page 1 never rendered results). - soft_blocked = result.blocked or (result.success and not result.hotels) + # Soft-block signals: an explicit challenge, a "successful" load + # that yielded zero hotels, or a failed load (page.content crash, + # timeout with no HTML) — all warrant an IP rotation when proxied. + soft_blocked = result.blocked or not result.hotels if not soft_blocked: return result @@ -635,8 +636,13 @@ class PlaywrightLocalBackend(ScraperBackend): logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False - # Check for blocking - content = await page.content() + # Check for blocking — page may still be navigating after a + # timeout, so wrap in try/except to avoid crashing the attempt. + try: + content = await page.content() + except Exception as ce: + logger.warning(f"page.content() unavailable (page still navigating): {ce}") + content = "" is_blocked, reason = self.detect_blocking(content) if is_blocked: logger.warning(f"Blocking detected: {reason}")