From b44e688b3e7fe1d889e054cf8fb40c7a8df7a6d8 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 10 Jul 2026 02:19:16 +0000 Subject: [PATCH] Retry page load timeouts: treat as blocks, bump max retries to 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DataImpulse endpoints are intermittently broken — some return EOF after CONNECT immediately, others work fine. DNS round-robins between them so the same request can fail 3× then succeed on the 4th. Timeouts were returning blocked=False and silently failing with no retry. Now treated as retryable blocks (with context rotation) so a fresh endpoint is tried. Max retries 2→4 (5 total attempts) to give enough chances to land a working DataImpulse endpoint. Co-Authored-By: Claude Sonnet 4.6 --- backend/services/booking_scraper.py | 2 +- backend/services/scraper_backends/playwright_hotel_page.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 7a1e48a..32e8cc8 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -318,7 +318,7 @@ async def scrape_hotel_date( backend: 'PlaywrightHotelPageBackend', batch_id: uuid.UUID, adults: int = 2, - max_retries: int = 2, + max_retries: int = 4, ) -> Dict[str, Any]: """ Scrape all rate plans for one hotel on one date via the hotel page backend. diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py index 387ccbb..b7ecb4d 100644 --- a/backend/services/scraper_backends/playwright_hotel_page.py +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -298,7 +298,11 @@ class PlaywrightHotelPageBackend(ScraperBackend): logger.warning(f"goto timeout for {hotel_url} {check_in}: {e}") if not page_loaded: - return ScraperResult(success=False, blocked=False, error_message='page load timeout') + # Treat as a retryable block — DataImpulse endpoints are intermittently + # broken (EOF after CONNECT); rotating to a fresh context hits a different + # endpoint and usually succeeds within 1-2 retries. + await self._rotate_context() + return ScraperResult(success=False, blocked=True, block_reason='page load timeout') # Wait for the room table (prices render via JS after DOM) room_table_appeared = False