Add direct connection fallback after all proxy attempts fail
When all 3 proxy rotation attempts are soft-blocked (bad IP pool), fall back to the server's own IP for a final attempt. Keeps proxy as primary for IP diversity / rate-limit protection while ensuring a clean fallback when the assigned residential pool is Cloudflare-challenged. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
7b4bb72f15
commit
2ed53bea39
1 changed files with 16 additions and 0 deletions
|
|
@ -550,6 +550,22 @@ class PlaywrightLocalBackend(ScraperBackend):
|
||||||
await self.rotate_session()
|
await self.rotate_session()
|
||||||
await asyncio.sleep(random.uniform(1.0, 2.0))
|
await asyncio.sleep(random.uniform(1.0, 2.0))
|
||||||
|
|
||||||
|
# All proxy attempts exhausted — fall back to direct connection. The
|
||||||
|
# server's own IP may be cleaner than the proxy pool currently assigned.
|
||||||
|
if self._proxy_enabled():
|
||||||
|
logger.warning(
|
||||||
|
f"All {max_attempts} proxy attempt(s) failed for {check_in} — "
|
||||||
|
"falling back to direct connection"
|
||||||
|
)
|
||||||
|
saved_proxy = self.proxy_config
|
||||||
|
self.proxy_config = {}
|
||||||
|
await self.rotate_session() # closes proxy browser so next _ensure_context starts fresh
|
||||||
|
try:
|
||||||
|
result = await self._scrape_once(
|
||||||
|
location, check_in, check_out, adults, pages, dest_id, location_params)
|
||||||
|
finally:
|
||||||
|
self.proxy_config = saved_proxy
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def _scrape_once(
|
async def _scrape_once(
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue