hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Fix page.content() crash and ensure rotation triggers on any empty result

Browse source 
page.content() throws when page is still navigating after a goto timeout;
catch it and continue with empty content. Also broaden the soft-block check
to trigger IP rotation whenever no hotels are found (not just on explicit
blocks), so proxy rotation fires even when the error path is hit.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 18:16:06 +00:00
parent 2ed53bea39
commit 92dce29e7b
1 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

14
backend/services/scraper_backends/playwright_local.py
View file

@ -536,9 +536,10 @@ class PlaywrightLocalBackend(ScraperBackend):
result = await self._scrape_once( result = await self._scrape_once(
location, check_in, check_out, adults, pages, dest_id, location_params) location, check_in, check_out, adults, pages, dest_id, location_params)
# Soft-block signals: an explicit challenge, or a "successful" load # Soft-block signals: an explicit challenge, a "successful" load
# that yielded zero hotels (page 1 never rendered results). # that yielded zero hotels, or a failed load (page.content crash,
soft_blocked = result.blocked or (result.success and not result.hotels) # timeout with no HTML) — all warrant an IP rotation when proxied.
soft_blocked = result.blocked or not result.hotels
if not soft_blocked: if not soft_blocked:
return result return result
@ -635,8 +636,13 @@ class PlaywrightLocalBackend(ScraperBackend):
logger.warning(f"Page load timeout, continuing: {e}") logger.warning(f"Page load timeout, continuing: {e}")
page_loaded = False page_loaded = False
# Check for blocking # Check for blocking — page may still be navigating after a
# timeout, so wrap in try/except to avoid crashing the attempt.
try:
content = await page.content() content = await page.content()
except Exception as ce:
logger.warning(f"page.content() unavailable (page still navigating): {ce}")
content = ""
is_blocked, reason = self.detect_blocking(content) is_blocked, reason = self.detect_blocking(content)
if is_blocked: if is_blocked:
logger.warning(f"Blocking detected: {reason}") logger.warning(f"Blocking detected: {reason}")