Add direct connection fallback after all proxy attempts fail

When all 3 proxy rotation attempts are soft-blocked (bad IP pool), fall
back to the server's own IP for a final attempt. Keeps proxy as primary
for IP diversity / rate-limit protection while ensuring a clean fallback
when the assigned residential pool is Cloudflare-challenged.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 18:01:46 +00:00
parent 7b4bb72f15
commit 2ed53bea39

View file

@ -550,6 +550,22 @@ class PlaywrightLocalBackend(ScraperBackend):
await self.rotate_session() await self.rotate_session()
await asyncio.sleep(random.uniform(1.0, 2.0)) await asyncio.sleep(random.uniform(1.0, 2.0))
# All proxy attempts exhausted — fall back to direct connection. The
# server's own IP may be cleaner than the proxy pool currently assigned.
if self._proxy_enabled():
logger.warning(
f"All {max_attempts} proxy attempt(s) failed for {check_in}"
"falling back to direct connection"
)
saved_proxy = self.proxy_config
self.proxy_config = {}
await self.rotate_session() # closes proxy browser so next _ensure_context starts fresh
try:
result = await self._scrape_once(
location, check_in, check_out, adults, pages, dest_id, location_params)
finally:
self.proxy_config = saved_proxy
return result return result
async def _scrape_once( async def _scrape_once(