jtricerolph
134cc5cc3f
DataImpulse takes ~14s to issue a 407 challenge from this network. Passing credentials as separate Playwright proxy fields caused Chromium to wait for that challenge before sending auth on every CONNECT request, making scrapes consistently time out at 90s. Embedding credentials directly in the proxy server URL makes Chromium send Proxy-Authorization on the first CONNECT, bypassing the round-trip. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
116 lines
4.6 KiB
Python
116 lines
4.6 KiB
Python
"""
|
|
Shared residential-proxy configuration for every scraper.
|
|
|
|
Single source of truth for:
|
|
- where proxy config lives (system_config `booking_proxy_*` keys, with
|
|
BOOKING_PROXY_* env fallback),
|
|
- the DataImpulse sticky-session username syntax (LOGIN__cr.<cc>;sessid.<id>),
|
|
- building proxy settings for both Playwright (launch dict) and httpx (URL).
|
|
|
|
The Booking.com browser scraper (Playwright) and the direct booking-engine
|
|
scraper (httpx) both consume this, so proxy logic is never duplicated. Direct
|
|
scraping is opt-in via the `direct_scraper_use_proxy` config key (default off —
|
|
booking-engine APIs generally don't need it), but the plumbing is ready.
|
|
"""
|
|
import os
|
|
import random
|
|
from typing import Optional
|
|
|
|
|
|
def new_session_id() -> str:
|
|
"""A fresh sticky-session id. DataImpulse pins one IP per id, so a new id
|
|
yields a new residential IP."""
|
|
return f"hnf{random.randint(100000, 999999)}"
|
|
|
|
|
|
def config_from_env() -> dict:
|
|
"""Proxy config from BOOKING_PROXY_* env vars ({} when unset)."""
|
|
host = os.getenv("BOOKING_PROXY_HOST", "").strip()
|
|
if not host:
|
|
return {}
|
|
return {
|
|
'host': host,
|
|
'port': os.getenv("BOOKING_PROXY_PORT", "823").strip(),
|
|
'username': os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
|
|
'password': os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
|
|
'country': os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
|
|
}
|
|
|
|
|
|
def normalize(raw: dict) -> dict:
|
|
"""Turn raw `booking_proxy_*` system_config values into a proxy config dict.
|
|
DB is authoritative when `booking_proxy_enabled` is present; otherwise fall
|
|
back to env. Returns {} when disabled or unconfigured."""
|
|
if 'booking_proxy_enabled' in raw:
|
|
if raw.get('booking_proxy_enabled') != 'true':
|
|
return {}
|
|
return {
|
|
'host': (raw.get('booking_proxy_host') or '').strip(),
|
|
'port': (raw.get('booking_proxy_port') or '823').strip(),
|
|
'username': (raw.get('booking_proxy_username') or '').strip(),
|
|
'password': (raw.get('booking_proxy_password') or '').strip(),
|
|
'country': (raw.get('booking_proxy_country') or 'gb').strip(),
|
|
}
|
|
return config_from_env()
|
|
|
|
|
|
def load_config(db) -> dict:
|
|
"""Resolve proxy config using a sync SQLAlchemy session. Returns {} when
|
|
disabled/unconfigured. (Async callers should fetch the rows themselves and
|
|
pass them through normalize().)"""
|
|
from sqlalchemy import text
|
|
rows = db.execute(
|
|
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
|
|
).fetchall()
|
|
return normalize({r.config_key: r.config_value for r in rows})
|
|
|
|
|
|
def is_enabled(cfg: dict) -> bool:
|
|
return bool(cfg.get('host') and cfg.get('username'))
|
|
|
|
|
|
def username(cfg: dict, session_id: Optional[str] = None) -> str:
|
|
"""DataImpulse username: LOGIN__cr.<country>[;sessid.<id>]."""
|
|
user = f"{cfg['username']}__cr.{cfg.get('country', 'gb')}"
|
|
if session_id:
|
|
user += f";sessid.{session_id}"
|
|
return user
|
|
|
|
|
|
def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]:
|
|
"""Proxy dict for chromium.launch(proxy=...). None when disabled.
|
|
|
|
Credentials are embedded in the server URL rather than passed as separate
|
|
fields. Separate fields cause Chromium to wait for a 407 challenge before
|
|
sending auth — DataImpulse takes ~14s to issue that challenge, making every
|
|
page.goto() timeout. Embedded credentials are sent on the first CONNECT
|
|
request, bypassing the round-trip entirely.
|
|
"""
|
|
if not is_enabled(cfg):
|
|
return None
|
|
from urllib.parse import quote
|
|
user = quote(username(cfg, session_id), safe='')
|
|
pwd = quote(cfg['password'], safe='')
|
|
return {
|
|
'server': f"http://{user}:{pwd}@{cfg['host']}:{cfg['port']}",
|
|
}
|
|
|
|
|
|
def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]:
|
|
"""Proxy URL for httpx.AsyncClient(proxies=...). None when disabled."""
|
|
if not is_enabled(cfg):
|
|
return None
|
|
return f"http://{username(cfg, session_id)}:{cfg['password']}@{cfg['host']}:{cfg['port']}"
|
|
|
|
|
|
def direct_httpx_proxy(db) -> Optional[str]:
|
|
"""Proxy URL for the direct booking-engine scraper — only when the proxy is
|
|
configured AND `direct_scraper_use_proxy` is explicitly enabled (default off).
|
|
Uses a fresh session id per call so direct runs spread across IPs."""
|
|
from sqlalchemy import text
|
|
flag = db.execute(
|
|
text("SELECT config_value FROM system_config WHERE config_key = 'direct_scraper_use_proxy'")
|
|
).fetchone()
|
|
if not flag or flag.config_value != 'true':
|
|
return None
|
|
return httpx_proxy_url(load_config(db), new_session_id())
|