From 92dce29e7b6599387180b68539d73d2c45f2db5d Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 18:16:06 +0000 Subject: [PATCH] Fix page.content() crash and ensure rotation triggers on any empty result page.content() throws when page is still navigating after a goto timeout; catch it and continue with empty content. Also broaden the soft-block check to trigger IP rotation whenever no hotels are found (not just on explicit blocks), so proxy rotation fires even when the error path is hit. Co-Authored-By: Claude Sonnet 4.6 --- .../scraper_backends/playwright_local.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 660ec84..698f4c4 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -536,9 +536,10 @@ class PlaywrightLocalBackend(ScraperBackend): result = await self._scrape_once( location, check_in, check_out, adults, pages, dest_id, location_params) - # Soft-block signals: an explicit challenge, or a "successful" load - # that yielded zero hotels (page 1 never rendered results). - soft_blocked = result.blocked or (result.success and not result.hotels) + # Soft-block signals: an explicit challenge, a "successful" load + # that yielded zero hotels, or a failed load (page.content crash, + # timeout with no HTML) — all warrant an IP rotation when proxied. + soft_blocked = result.blocked or not result.hotels if not soft_blocked: return result @@ -635,8 +636,13 @@ class PlaywrightLocalBackend(ScraperBackend): logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False - # Check for blocking - content = await page.content() + # Check for blocking — page may still be navigating after a + # timeout, so wrap in try/except to avoid crashing the attempt. + try: + content = await page.content() + except Exception as ce: + logger.warning(f"page.content() unavailable (page still navigating): {ce}") + content = "" is_blocked, reason = self.detect_blocking(content) if is_blocked: logger.warning(f"Blocking detected: {reason}")