From 2ed53bea396d8ca3a78594a7d58934a9854b31c0 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 18:01:46 +0000 Subject: [PATCH] Add direct connection fallback after all proxy attempts fail When all 3 proxy rotation attempts are soft-blocked (bad IP pool), fall back to the server's own IP for a final attempt. Keeps proxy as primary for IP diversity / rate-limit protection while ensuring a clean fallback when the assigned residential pool is Cloudflare-challenged. Co-Authored-By: Claude Sonnet 4.6 --- .../scraper_backends/playwright_local.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 710e6d2..660ec84 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -550,6 +550,22 @@ class PlaywrightLocalBackend(ScraperBackend): await self.rotate_session() await asyncio.sleep(random.uniform(1.0, 2.0)) + # All proxy attempts exhausted — fall back to direct connection. The + # server's own IP may be cleaner than the proxy pool currently assigned. + if self._proxy_enabled(): + logger.warning( + f"All {max_attempts} proxy attempt(s) failed for {check_in} — " + "falling back to direct connection" + ) + saved_proxy = self.proxy_config + self.proxy_config = {} + await self.rotate_session() # closes proxy browser so next _ensure_context starts fresh + try: + result = await self._scrape_once( + location, check_in, check_out, adults, pages, dest_id, location_params) + finally: + self.proxy_config = saved_proxy + return result async def _scrape_once(