From 940ffbbe3922fafad70b4690ff0e71e72204599f Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 10 Jul 2026 02:13:06 +0000 Subject: [PATCH] Switch proxy auth from URL-embedded to Proxy-Authorization header Embedding credentials in the proxy URL (http://user:pass@host:port) was breaking HTTPS CONNECT tunnels on the hotel network. Switching to separate username/password fields (Playwright) and httpx.Proxy(auth=...) sends a Proxy-Authorization header instead, which passes through correctly. With DataImpulse IP whitelisting the 407 round-trip is skipped anyway so there is no latency penalty. Co-Authored-By: Claude Sonnet 4.6 --- backend/api/competitors.py | 9 ++++---- backend/services/proxy.py | 45 ++++++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 02504fd..44f8fad 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -277,14 +277,13 @@ async def test_proxy_config( 'password': (raw.get('booking_proxy_password') or '').strip(), 'country': (raw.get('booking_proxy_country') or 'gb').strip(), } - if not (proxy_util.is_enabled(cfg) and cfg['password']): - raise HTTPException(status_code=400, detail="Proxy host, username and password must be saved first.") - - proxy_url = proxy_util.httpx_proxy_url(cfg, proxy_util.new_session_id()) + if not proxy_util.is_enabled(cfg): + raise HTTPException(status_code=400, detail="Proxy host and username must be saved first.") import httpx + proxy = proxy_util.httpx_proxy(cfg, proxy_util.new_session_id()) try: - async with httpx.AsyncClient(proxy=proxy_url, timeout=40.0) as client: + async with httpx.AsyncClient(proxy=proxy, timeout=40.0) as client: resp = await client.get("https://ipinfo.io/json") resp.raise_for_status() data = resp.json() diff --git a/backend/services/proxy.py b/backend/services/proxy.py index 8cdbcde..c4ced2b 100644 --- a/backend/services/proxy.py +++ b/backend/services/proxy.py @@ -78,33 +78,46 @@ def username(cfg: dict, session_id: Optional[str] = None) -> str: def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]: - """Proxy dict for chromium.launch(proxy=...). None when disabled. + """Proxy dict for new_context(proxy=...). None when disabled. - Credentials are embedded in the server URL rather than passed as separate - fields. Separate fields cause Chromium to wait for a 407 challenge before - sending auth — DataImpulse takes ~14s to issue that challenge, making every - page.goto() timeout. Embedded credentials are sent on the first CONNECT - request, bypassing the round-trip entirely. + Uses separate username/password fields so Chromium sends a + Proxy-Authorization header rather than embedding credentials in the URL. + With IP whitelisting on DataImpulse, the 407 round-trip is skipped entirely + (the proxy accepts on IP alone), so there is no speed penalty. """ if not is_enabled(cfg): return None - from urllib.parse import quote - user = quote(username(cfg, session_id), safe='') - pwd = quote(cfg['password'], safe='') return { - 'server': f"http://{user}:{pwd}@{cfg['host']}:{cfg['port']}", + 'server': f"http://{cfg['host']}:{cfg['port']}", + 'username': username(cfg, session_id), + 'password': cfg.get('password', ''), } -def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]: - """Proxy URL for httpx.AsyncClient(proxies=...). None when disabled.""" +def httpx_proxy(cfg: dict, session_id: Optional[str] = None): + """httpx.Proxy object for AsyncClient(proxy=...). None when disabled. + + Uses the auth= kwarg so credentials are sent as a Proxy-Authorization + header rather than embedded in the URL. + """ if not is_enabled(cfg): return None - return f"http://{username(cfg, session_id)}:{cfg['password']}@{cfg['host']}:{cfg['port']}" + import httpx + return httpx.Proxy( + f"http://{cfg['host']}:{cfg['port']}", + auth=(username(cfg, session_id), cfg.get('password', '')), + ) -def direct_httpx_proxy(db) -> Optional[str]: - """Proxy URL for the direct booking-engine scraper — only when the proxy is +def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]: + """Legacy URL form — prefer httpx_proxy() for new callers.""" + if not is_enabled(cfg): + return None + return f"http://{cfg['host']}:{cfg['port']}" + + +def direct_httpx_proxy(db): + """httpx.Proxy for the direct booking-engine scraper — only when the proxy is configured AND `direct_scraper_use_proxy` is explicitly enabled (default off). Uses a fresh session id per call so direct runs spread across IPs.""" from sqlalchemy import text @@ -113,4 +126,4 @@ def direct_httpx_proxy(db) -> Optional[str]: ).fetchone() if not flag or flag.config_value != 'true': return None - return httpx_proxy_url(load_config(db), new_session_id()) + return httpx_proxy(load_config(db), new_session_id())