jtricerolph
28c4dd9f54
Header-based auth made no difference; the real issue is the hotel network firewall blocking HTTPS CONNECT tunnels on port 823. Reverted to URL-embedded credentials (original approach). Fix requires DataImpulse to enable port 443. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
124 lines
4.9 KiB
Python
124 lines
4.9 KiB
Python
"""
|
|
Shared residential-proxy configuration for every scraper.
|
|
|
|
Single source of truth for:
|
|
- where proxy config lives (system_config `booking_proxy_*` keys, with
|
|
BOOKING_PROXY_* env fallback),
|
|
- the DataImpulse sticky-session username syntax (LOGIN__cr.<cc>;sessid.<id>),
|
|
- building proxy settings for both Playwright (launch dict) and httpx (URL).
|
|
|
|
The Booking.com browser scraper (Playwright) and the direct booking-engine
|
|
scraper (httpx) both consume this, so proxy logic is never duplicated. Direct
|
|
scraping is opt-in via the `direct_scraper_use_proxy` config key (default off —
|
|
booking-engine APIs generally don't need it), but the plumbing is ready.
|
|
"""
|
|
import os
|
|
import random
|
|
from typing import Optional
|
|
|
|
|
|
def new_session_id() -> str:
|
|
"""A fresh sticky-session id. DataImpulse pins one IP per id, so a new id
|
|
yields a new residential IP."""
|
|
return f"hnf{random.randint(100000, 999999)}"
|
|
|
|
|
|
def config_from_env() -> dict:
|
|
"""Proxy config from BOOKING_PROXY_* env vars ({} when unset)."""
|
|
host = os.getenv("BOOKING_PROXY_HOST", "").strip()
|
|
if not host:
|
|
return {}
|
|
return {
|
|
'host': host,
|
|
'port': os.getenv("BOOKING_PROXY_PORT", "823").strip(),
|
|
'username': os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
|
|
'password': os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
|
|
'country': os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
|
|
}
|
|
|
|
|
|
def normalize(raw: dict) -> dict:
|
|
"""Turn raw `booking_proxy_*` system_config values into a proxy config dict.
|
|
DB is authoritative when `booking_proxy_enabled` is present; otherwise fall
|
|
back to env. Returns {} when disabled or unconfigured."""
|
|
if 'booking_proxy_enabled' in raw:
|
|
if raw.get('booking_proxy_enabled') != 'true':
|
|
return {}
|
|
return {
|
|
'host': (raw.get('booking_proxy_host') or '').strip(),
|
|
'port': (raw.get('booking_proxy_port') or '823').strip(),
|
|
'username': (raw.get('booking_proxy_username') or '').strip(),
|
|
'password': (raw.get('booking_proxy_password') or '').strip(),
|
|
'country': (raw.get('booking_proxy_country') or 'gb').strip(),
|
|
}
|
|
return config_from_env()
|
|
|
|
|
|
def load_config(db) -> dict:
|
|
"""Resolve proxy config using a sync SQLAlchemy session. Returns {} when
|
|
disabled/unconfigured. (Async callers should fetch the rows themselves and
|
|
pass them through normalize().)"""
|
|
from sqlalchemy import text
|
|
rows = db.execute(
|
|
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
|
|
).fetchall()
|
|
return normalize({r.config_key: r.config_value for r in rows})
|
|
|
|
|
|
def is_enabled(cfg: dict) -> bool:
|
|
return bool(cfg.get('host') and cfg.get('username'))
|
|
|
|
|
|
def username(cfg: dict, session_id: Optional[str] = None) -> str:
|
|
"""DataImpulse username: LOGIN__cr.<country>[;sessid.<id>]."""
|
|
user = f"{cfg['username']}__cr.{cfg.get('country', 'gb')}"
|
|
if session_id:
|
|
user += f";sessid.{session_id}"
|
|
return user
|
|
|
|
|
|
def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]:
|
|
"""Proxy dict for new_context(proxy=...). None when disabled.
|
|
|
|
Credentials are embedded in the server URL rather than passed as separate
|
|
fields. Separate fields cause Chromium to wait for a 407 challenge before
|
|
sending auth — DataImpulse takes ~14s to issue that challenge, making every
|
|
page.goto() timeout. Embedded credentials are sent on the first CONNECT
|
|
request, bypassing the round-trip entirely.
|
|
"""
|
|
if not is_enabled(cfg):
|
|
return None
|
|
from urllib.parse import quote
|
|
user = quote(username(cfg, session_id), safe='')
|
|
pwd = quote(cfg.get('password', ''), safe='')
|
|
return {
|
|
'server': f"http://{user}:{pwd}@{cfg['host']}:{cfg['port']}",
|
|
}
|
|
|
|
|
|
def httpx_proxy(cfg: dict, session_id: Optional[str] = None):
|
|
"""httpx.Proxy object for AsyncClient(proxy=...). None when disabled."""
|
|
if not is_enabled(cfg):
|
|
return None
|
|
import httpx
|
|
return httpx.Proxy(httpx_proxy_url(cfg, session_id))
|
|
|
|
|
|
def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]:
|
|
"""Proxy URL for httpx.AsyncClient(proxy=...). None when disabled."""
|
|
if not is_enabled(cfg):
|
|
return None
|
|
return f"http://{username(cfg, session_id)}:{cfg.get('password', '')}@{cfg['host']}:{cfg['port']}"
|
|
|
|
|
|
def direct_httpx_proxy(db):
|
|
"""httpx.Proxy for the direct booking-engine scraper — only when the proxy is
|
|
configured AND `direct_scraper_use_proxy` is explicitly enabled (default off).
|
|
Uses a fresh session id per call so direct runs spread across IPs."""
|
|
from sqlalchemy import text
|
|
flag = db.execute(
|
|
text("SELECT config_value FROM system_config WHERE config_key = 'direct_scraper_use_proxy'")
|
|
).fetchone()
|
|
if not flag or flag.config_value != 'true':
|
|
return None
|
|
return httpx_proxy(load_config(db), new_session_id())
|