rates/backend/services/proxy.py

"""
Shared residential-proxy configuration for every scraper.

Single source of truth for:
  - where proxy config lives (system_config `booking_proxy_*` keys, with
    BOOKING_PROXY_* env fallback),
  - the DataImpulse sticky-session username syntax (LOGIN__cr.<cc>;sessid.<id>),
  - building proxy settings for both Playwright (launch dict) and httpx (URL).

The Booking.com browser scraper (Playwright) and the direct booking-engine
scraper (httpx) both consume this, so proxy logic is never duplicated. Direct
scraping is opt-in via the `direct_scraper_use_proxy` config key (default off —
booking-engine APIs generally don't need it), but the plumbing is ready.
"""
import os
import random
from typing import Optional


def new_session_id() -> str:
    """A fresh sticky-session id. DataImpulse pins one IP per id, so a new id
    yields a new residential IP."""
    return f"hnf{random.randint(100000, 999999)}"


def config_from_env() -> dict:
    """Proxy config from BOOKING_PROXY_* env vars ({} when unset)."""
    host = os.getenv("BOOKING_PROXY_HOST", "").strip()
    if not host:
        return {}
    return {
        'host': host,
        'port': os.getenv("BOOKING_PROXY_PORT", "823").strip(),
        'username': os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
        'password': os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
        'country': os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
    }


def normalize(raw: dict) -> dict:
    """Turn raw `booking_proxy_*` system_config values into a proxy config dict.
    DB is authoritative when `booking_proxy_enabled` is present; otherwise fall
    back to env. Returns {} when disabled or unconfigured."""
    if 'booking_proxy_enabled' in raw:
        if raw.get('booking_proxy_enabled') != 'true':
            return {}
        return {
            'host': (raw.get('booking_proxy_host') or '').strip(),
            'port': (raw.get('booking_proxy_port') or '823').strip(),
            'username': (raw.get('booking_proxy_username') or '').strip(),
            'password': (raw.get('booking_proxy_password') or '').strip(),
            'country': (raw.get('booking_proxy_country') or 'gb').strip(),
        }
    return config_from_env()


def load_config(db) -> dict:
    """Resolve proxy config using a sync SQLAlchemy session. Returns {} when
    disabled/unconfigured. (Async callers should fetch the rows themselves and
    pass them through normalize().)"""
    from sqlalchemy import text
    rows = db.execute(
        text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
    ).fetchall()
    return normalize({r.config_key: r.config_value for r in rows})


def is_enabled(cfg: dict) -> bool:
    return bool(cfg.get('host') and cfg.get('username'))


def username(cfg: dict, session_id: Optional[str] = None) -> str:
    """DataImpulse username: LOGIN__cr.<country>[;sessid.<id>]."""
    user = f"{cfg['username']}__cr.{cfg.get('country', 'gb')}"
    if session_id:
        user += f";sessid.{session_id}"
    return user


def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]:
    """Proxy dict for new_context(proxy=...). None when disabled.

    Uses separate username/password fields so Chromium sends a
    Proxy-Authorization header rather than embedding credentials in the URL.
    With IP whitelisting on DataImpulse, the 407 round-trip is skipped entirely
    (the proxy accepts on IP alone), so there is no speed penalty.
    """
    if not is_enabled(cfg):
        return None
    return {
        'server': f"http://{cfg['host']}:{cfg['port']}",
        'username': username(cfg, session_id),
        'password': cfg.get('password', ''),
    }


def httpx_proxy(cfg: dict, session_id: Optional[str] = None):
    """httpx.Proxy object for AsyncClient(proxy=...). None when disabled.

    Uses the auth= kwarg so credentials are sent as a Proxy-Authorization
    header rather than embedded in the URL.
    """
    if not is_enabled(cfg):
        return None
    import httpx
    return httpx.Proxy(
        f"http://{cfg['host']}:{cfg['port']}",
        auth=(username(cfg, session_id), cfg.get('password', '')),
    )


def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]:
    """Legacy URL form — prefer httpx_proxy() for new callers."""
    if not is_enabled(cfg):
        return None
    return f"http://{cfg['host']}:{cfg['port']}"


def direct_httpx_proxy(db):
    """httpx.Proxy for the direct booking-engine scraper — only when the proxy is
    configured AND `direct_scraper_use_proxy` is explicitly enabled (default off).
    Uses a fresh session id per call so direct runs spread across IPs."""
    from sqlalchemy import text
    flag = db.execute(
        text("SELECT config_value FROM system_config WHERE config_key = 'direct_scraper_use_proxy'")
    ).fetchone()
    if not flag or flag.config_value != 'true':
        return None
    return httpx_proxy(load_config(db), new_session_id())