Add direct connection fallback after all proxy attempts fail
When all 3 proxy rotation attempts are soft-blocked (bad IP pool), fall back to the server's own IP for a final attempt. Keeps proxy as primary for IP diversity / rate-limit protection while ensuring a clean fallback when the assigned residential pool is Cloudflare-challenged. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
7b4bb72f15
commit
2ed53bea39
1 changed files with 16 additions and 0 deletions
|
|
@ -550,6 +550,22 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
await self.rotate_session()
|
||||
await asyncio.sleep(random.uniform(1.0, 2.0))
|
||||
|
||||
# All proxy attempts exhausted — fall back to direct connection. The
|
||||
# server's own IP may be cleaner than the proxy pool currently assigned.
|
||||
if self._proxy_enabled():
|
||||
logger.warning(
|
||||
f"All {max_attempts} proxy attempt(s) failed for {check_in} — "
|
||||
"falling back to direct connection"
|
||||
)
|
||||
saved_proxy = self.proxy_config
|
||||
self.proxy_config = {}
|
||||
await self.rotate_session() # closes proxy browser so next _ensure_context starts fresh
|
||||
try:
|
||||
result = await self._scrape_once(
|
||||
location, check_in, check_out, adults, pages, dest_id, location_params)
|
||||
finally:
|
||||
self.proxy_config = saved_proxy
|
||||
|
||||
return result
|
||||
|
||||
async def _scrape_once(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue