From 134cc5cc3ff5f7a9d96b5e14674f0b4bf2306ea6 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 20:14:09 +0000 Subject: [PATCH] Fix proxy 407 round-trip causing 14s latency per page.goto() DataImpulse takes ~14s to issue a 407 challenge from this network. Passing credentials as separate Playwright proxy fields caused Chromium to wait for that challenge before sending auth on every CONNECT request, making scrapes consistently time out at 90s. Embedding credentials directly in the proxy server URL makes Chromium send Proxy-Authorization on the first CONNECT, bypassing the round-trip. Co-Authored-By: Claude Sonnet 4.6 --- backend/services/proxy.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/backend/services/proxy.py b/backend/services/proxy.py index ea8e30a..8cdbcde 100644 --- a/backend/services/proxy.py +++ b/backend/services/proxy.py @@ -78,13 +78,21 @@ def username(cfg: dict, session_id: Optional[str] = None) -> str: def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]: - """Proxy dict for chromium.launch(proxy=...). None when disabled.""" + """Proxy dict for chromium.launch(proxy=...). None when disabled. + + Credentials are embedded in the server URL rather than passed as separate + fields. Separate fields cause Chromium to wait for a 407 challenge before + sending auth — DataImpulse takes ~14s to issue that challenge, making every + page.goto() timeout. Embedded credentials are sent on the first CONNECT + request, bypassing the round-trip entirely. + """ if not is_enabled(cfg): return None + from urllib.parse import quote + user = quote(username(cfg, session_id), safe='') + pwd = quote(cfg['password'], safe='') return { - 'server': f"http://{cfg['host']}:{cfg['port']}", - 'username': username(cfg, session_id), - 'password': cfg['password'], + 'server': f"http://{user}:{pwd}@{cfg['host']}:{cfg['port']}", }