diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 0d0ff1a..7ce33a1 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -636,13 +636,15 @@ class PlaywrightLocalBackend(ScraperBackend): logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False - # Check for blocking — page may still be navigating after a - # timeout, so wrap in try/except to avoid crashing the attempt. - try: - content = await page.content() - except Exception as ce: - logger.warning(f"page.content() unavailable (page still navigating): {ce}") - content = "" + # Check for blocking. Skip when the page never loaded — page.content() + # blocks until any in-progress navigation (e.g. Cloudflare redirect) + # completes before raising, adding minutes of hidden delay per attempt. + content = "" + if page_loaded: + try: + content = await page.content() + except Exception as ce: + logger.warning(f"page.content() unavailable: {ce}") is_blocked, reason = self.detect_blocking(content) if is_blocked: logger.warning(f"Blocking detected: {reason}") @@ -679,6 +681,12 @@ class PlaywrightLocalBackend(ScraperBackend): logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique") + # Page 1 failed to load and returned nothing — the browser is in + # a broken/challenge state; trying page 2 just wastes another 60s. + if page_num == 0 and not page_loaded and not all_hotels: + logger.warning("Page 1 load failed with 0 hotels — skipping page 2") + break + return ScraperResult( success=True, blocked=False,