Extract shared proxy module; make it available to direct scraper; remove dead code

Proxy config, DataImpulse sticky-session username syntax, and Playwright/
httpx proxy builders now live in one place (services/proxy.py) instead of
being duplicated across the Booking.com backend and the /config/proxy test
endpoint. Both scrapers consume it.

- services/proxy.py: load_config/normalize (DB-authoritative, env fallback),
  new_session_id, username, playwright_proxy, httpx_proxy_url
- PlaywrightLocalBackend delegates proxy building to the module
- get_scraper_backend factory uses proxy.load_config (one resolution path)
- test_proxy_config endpoint uses the shared URL builder; httpx proxies= ->
  proxy= (forward-compatible, 0.28-safe)
- Direct booking-engine scraper (httpx) can now route through the same proxy,
  gated by the direct_scraper_use_proxy flag (default off, plumbing ready)

Dead code removed: set_scraper_paused (never called — rotate-on-block
replaced pause-on-block), get_competitor_matrix / get_hotels_list /
update_hotel_tier (endpoints have their own SQL), unused PROXY_KEYS tuple.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/api/competitors.py

@@ -12,6 +12,7 @@ import logging
 
 from database import get_db, SyncSessionLocal
 from auth import get_current_user
+from services import proxy as proxy_util
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
@@ -188,12 +189,6 @@ async def set_system_config(
 
 # ── Booking.com scraper proxy config ─────────────────────────────────────────
 
-PROXY_KEYS = (
-    'booking_proxy_enabled', 'booking_proxy_host', 'booking_proxy_port',
-    'booking_proxy_username', 'booking_proxy_password', 'booking_proxy_country',
-)
-
-
 class ProxyConfig(BaseModel):
     enabled: bool = False
     host: str = ''
@@ -265,22 +260,24 @@ async def test_proxy_config(
     result = await db.execute(
         text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
     )
-    cfg = {row.config_key: row.config_value for row in result.fetchall()}
+    raw = {row.config_key: row.config_value for row in result.fetchall()}
-
+    # Test whatever creds are stored, regardless of the enabled toggle, so the
-    host = (cfg.get('booking_proxy_host') or '').strip()
+    # user can verify before switching the proxy on.
-    username = (cfg.get('booking_proxy_username') or '').strip()
+    cfg = {
-    password = (cfg.get('booking_proxy_password') or '').strip()
+        'host': (raw.get('booking_proxy_host') or '').strip(),
-    if not (host and username and password):
+        'port': (raw.get('booking_proxy_port') or '823').strip(),
+        'username': (raw.get('booking_proxy_username') or '').strip(),
+        'password': (raw.get('booking_proxy_password') or '').strip(),
+        'country': (raw.get('booking_proxy_country') or 'gb').strip(),
+    }
+    if not (proxy_util.is_enabled(cfg) and cfg['password']):
         raise HTTPException(status_code=400, detail="Proxy host, username and password must be saved first.")
 
-    port = (cfg.get('booking_proxy_port') or '823').strip()
+    proxy_url = proxy_util.httpx_proxy_url(cfg, proxy_util.new_session_id())
-    country = (cfg.get('booking_proxy_country') or 'gb').strip()
-    proxy_user = f"{username}__cr.{country};sessid.hnftest"
-    proxy_url = f"http://{proxy_user}:{password}@{host}:{port}"
 
     import httpx
     try:
-        async with httpx.AsyncClient(proxies=proxy_url, timeout=40.0) as client:
+        async with httpx.AsyncClient(proxy=proxy_url, timeout=40.0) as client:
             resp = await client.get("https://ipinfo.io/json")
             resp.raise_for_status()
             data = resp.json()

backend/services/booking_scraper.py

@@ -23,6 +23,7 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from database import SyncSessionLocal
+from services import proxy as proxy_util
 from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
 
 logger = logging.getLogger(__name__)
@@ -53,31 +54,9 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
     if backend_type not in ('playwright_local', 'playwright_proxy'):
         logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
 
-    # Proxy configuration is managed in system_config (Settings page). When the
+    # Proxy config resolved by the shared module: system_config is authoritative
-    # 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
+    # when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct).
-    # the backend falls back to the BOOKING_PROXY_* environment variables.
+    return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db))
-    proxy_rows = db.execute(
-        text("""
-            SELECT config_key, config_value FROM system_config
-            WHERE config_key LIKE 'booking_proxy_%'
-        """)
-    ).fetchall()
-    proxy = {row.config_key: row.config_value for row in proxy_rows}
-
-    if 'booking_proxy_enabled' in proxy:
-        if proxy.get('booking_proxy_enabled') == 'true':
-            return PlaywrightLocalBackend(proxy_config={
-                'host': proxy.get('booking_proxy_host', ''),
-                'port': proxy.get('booking_proxy_port', '823'),
-                'username': proxy.get('booking_proxy_username', ''),
-                'password': proxy.get('booking_proxy_password', ''),
-                'country': proxy.get('booking_proxy_country', 'gb'),
-            })
-        # Explicitly disabled in the DB — direct connection, ignore env.
-        return PlaywrightLocalBackend(proxy_config={})
-
-    # No DB override — let the backend read BOOKING_PROXY_* env vars.
-    return PlaywrightLocalBackend()
 
 
 def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
@@ -135,21 +114,6 @@ async def is_scraper_paused(db: Session) -> bool:
     return True
 
 
-async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
-    """Set scraper pause status."""
-    db.execute(
-        text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
-        {'val': 'true' if paused else 'false'}
-    )
-    if paused:
-        pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
-        db.execute(
-            text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
-            {'val': pause_until}
-        )
-    db.commit()
-
-
 def save_hotel(db: Session, hotel: HotelData) -> int:
     """
     Save or update a hotel in the database.
@@ -861,105 +825,3 @@ async def _process_queue_locked(db: Session) -> Dict[str, Any]:
             'dates_failed': len(jobs),
         }
 
-
-def get_competitor_matrix(
-    db: Session,
-    from_date: date,
-    to_date: date,
-    include_market: bool = False
-) -> List[Dict[str, Any]]:
-    """
-    Get rate comparison matrix for competitors.
-
-    Args:
-        db: Database session
-        from_date: Start date
-        to_date: End date
-        include_market: Include market tier hotels
-
-    Returns:
-        List of rate records for matrix display
-    """
-    tier_filter = "h.tier IN ('own', 'competitor')"
-    if include_market:
-        tier_filter = "h.tier IN ('own', 'competitor', 'market')"
-
-    result = db.execute(
-        text(f"""
-            SELECT
-                r.rate_date,
-                h.id AS hotel_id,
-                h.name AS hotel_name,
-                h.tier,
-                h.display_order,
-                h.star_rating,
-                h.review_score,
-                r.availability_status,
-                r.rate_gross,
-                r.room_type,
-                r.breakfast_included,
-                r.free_cancellation,
-                r.no_prepayment,
-                r.rooms_left,
-                r.scraped_at
-            FROM booking_latest_rates r
-            JOIN booking_com_hotels h ON r.hotel_id = h.id
-            WHERE {tier_filter}
-              AND h.is_active = TRUE
-              AND r.rate_date BETWEEN :from_date AND :to_date
-            ORDER BY r.rate_date, h.display_order, h.name
-        """),
-        {'from_date': from_date, 'to_date': to_date}
-    )
-
-    return [dict(row._mapping) for row in result.fetchall()]
-
-
-def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
-    """
-    Get list of discovered hotels.
-
-    Args:
-        db: Database session
-        tier: Filter by tier ('own', 'competitor', 'market') or None for all
-
-    Returns:
-        List of hotel records
-    """
-    where_clause = "WHERE is_active = TRUE"
-    if tier:
-        where_clause += f" AND tier = '{tier}'"
-
-    result = db.execute(
-        text(f"""
-            SELECT
-                id, booking_com_id, name, booking_com_url,
-                star_rating, review_score, review_count,
-                tier, display_order, notes,
-                first_seen_at, last_seen_at
-            FROM booking_com_hotels
-            {where_clause}
-            ORDER BY display_order, name
-        """)
-    )
-
-    return [dict(row._mapping) for row in result.fetchall()]
-
-
-def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
-    """Update a hotel's tier and display order."""
-    if tier not in ('own', 'competitor', 'market'):
-        raise ValueError(f"Invalid tier: {tier}")
-
-    params = {'hotel_id': hotel_id, 'tier': tier}
-    set_clause = "tier = :tier"
-
-    if display_order is not None:
-        set_clause += ", display_order = :order"
-        params['order'] = display_order
-
-    db.execute(
-        text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
-        params
-    )
-    db.commit()

backend/services/direct_scraper.py

@@ -11,6 +11,7 @@ import httpx
 from sqlalchemy import text
 
 from database import SyncSessionLocal
+from services import proxy as proxy_util
 from services.direct_profiles import get_profile
 
 log = logging.getLogger(__name__)
@@ -18,6 +19,17 @@ log = logging.getLogger(__name__)
 REQUEST_DELAY = 10.0
 DISCOVERY_DELAY = 2.0
 
+
+def _direct_proxy_url() -> str | None:
+    """Proxy URL for direct-engine scraping, or None. Shares the Booking.com
+    proxy config but is gated by the `direct_scraper_use_proxy` flag (default
+    off — booking-engine APIs usually don't need it). Ready to switch on."""
+    db = SyncSessionLocal()
+    try:
+        return proxy_util.direct_httpx_proxy(db)
+    finally:
+        db.close()
+
 # Track running discovery scrapes: hotel_id -> status dict
 _discovery_status: dict[int, dict] = {}
 
@@ -50,7 +62,7 @@ async def run_discovery(hotel_id: int, profile_name: str, params: dict):
     found_rooms: set[str] = set()
     found_rates: set[str] = set()
 
-    async with httpx.AsyncClient() as client:
+    async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client:
         for i, arrival in enumerate(dates):
             await asyncio.sleep(DISCOVERY_DELAY)
             try:
@@ -127,7 +139,7 @@ def run_scrape(hotel_id: int, profile_name: str, params: dict):
 
 
 async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
-    async with httpx.AsyncClient() as client:
+    async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client:
         try:
             arrival_dates = await profile.fetch_arrival_dates(client, params)
         except Exception as e:

backend/services/proxy.py

@@ -0,0 +1,108 @@
+"""
+Shared residential-proxy configuration for every scraper.
+
+Single source of truth for:
+  - where proxy config lives (system_config `booking_proxy_*` keys, with
+    BOOKING_PROXY_* env fallback),
+  - the DataImpulse sticky-session username syntax (LOGIN__cr.<cc>;sessid.<id>),
+  - building proxy settings for both Playwright (launch dict) and httpx (URL).
+
+The Booking.com browser scraper (Playwright) and the direct booking-engine
+scraper (httpx) both consume this, so proxy logic is never duplicated. Direct
+scraping is opt-in via the `direct_scraper_use_proxy` config key (default off —
+booking-engine APIs generally don't need it), but the plumbing is ready.
+"""
+import os
+import random
+from typing import Optional
+
+
+def new_session_id() -> str:
+    """A fresh sticky-session id. DataImpulse pins one IP per id, so a new id
+    yields a new residential IP."""
+    return f"hnf{random.randint(100000, 999999)}"
+
+
+def config_from_env() -> dict:
+    """Proxy config from BOOKING_PROXY_* env vars ({} when unset)."""
+    host = os.getenv("BOOKING_PROXY_HOST", "").strip()
+    if not host:
+        return {}
+    return {
+        'host': host,
+        'port': os.getenv("BOOKING_PROXY_PORT", "823").strip(),
+        'username': os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
+        'password': os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
+        'country': os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
+    }
+
+
+def normalize(raw: dict) -> dict:
+    """Turn raw `booking_proxy_*` system_config values into a proxy config dict.
+    DB is authoritative when `booking_proxy_enabled` is present; otherwise fall
+    back to env. Returns {} when disabled or unconfigured."""
+    if 'booking_proxy_enabled' in raw:
+        if raw.get('booking_proxy_enabled') != 'true':
+            return {}
+        return {
+            'host': (raw.get('booking_proxy_host') or '').strip(),
+            'port': (raw.get('booking_proxy_port') or '823').strip(),
+            'username': (raw.get('booking_proxy_username') or '').strip(),
+            'password': (raw.get('booking_proxy_password') or '').strip(),
+            'country': (raw.get('booking_proxy_country') or 'gb').strip(),
+        }
+    return config_from_env()
+
+
+def load_config(db) -> dict:
+    """Resolve proxy config using a sync SQLAlchemy session. Returns {} when
+    disabled/unconfigured. (Async callers should fetch the rows themselves and
+    pass them through normalize().)"""
+    from sqlalchemy import text
+    rows = db.execute(
+        text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
+    ).fetchall()
+    return normalize({r.config_key: r.config_value for r in rows})
+
+
+def is_enabled(cfg: dict) -> bool:
+    return bool(cfg.get('host') and cfg.get('username'))
+
+
+def username(cfg: dict, session_id: Optional[str] = None) -> str:
+    """DataImpulse username: LOGIN__cr.<country>[;sessid.<id>]."""
+    user = f"{cfg['username']}__cr.{cfg.get('country', 'gb')}"
+    if session_id:
+        user += f";sessid.{session_id}"
+    return user
+
+
+def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]:
+    """Proxy dict for chromium.launch(proxy=...). None when disabled."""
+    if not is_enabled(cfg):
+        return None
+    return {
+        'server': f"http://{cfg['host']}:{cfg['port']}",
+        'username': username(cfg, session_id),
+        'password': cfg['password'],
+    }
+
+
+def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]:
+    """Proxy URL for httpx.AsyncClient(proxies=...). None when disabled."""
+    if not is_enabled(cfg):
+        return None
+    return f"http://{username(cfg, session_id)}:{cfg['password']}@{cfg['host']}:{cfg['port']}"
+
+
+def direct_httpx_proxy(db) -> Optional[str]:
+    """Proxy URL for the direct booking-engine scraper — only when the proxy is
+    configured AND `direct_scraper_use_proxy` is explicitly enabled (default off).
+    Uses a fresh session id per call so direct runs spread across IPs."""
+    from sqlalchemy import text
+    flag = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'direct_scraper_use_proxy'")
+    ).fetchone()
+    if not flag or flag.config_value != 'true':
+        return None
+    return httpx_proxy_url(load_config(db), new_session_id())

backend/services/scraper_backends/playwright_local.py

@@ -7,7 +7,6 @@ No proxy - direct connection. Suitable for low-volume scraping.
 
 import asyncio
 import logging
-import os
 import random
 import re
 from datetime import date
@@ -15,6 +14,8 @@ from decimal import Decimal, InvalidOperation
 from typing import List, Optional
 from urllib.parse import urlencode, urlparse, parse_qs
 
+from services import proxy as proxy_util
+
 
 def location_params_from_url(url: str) -> dict:
     """Pull the stable destination params (ss, dest_id, dest_type) out of a
@@ -109,8 +110,9 @@ class PlaywrightLocalBackend(ScraperBackend):
         Args:
             proxy_config: Optional proxy configuration. Falls back to the
                 BOOKING_PROXY_* environment variables when not supplied.
+                See services/proxy.py for the shared config/build helpers.
         """
-        self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env()
+        self.proxy_config = proxy_config if proxy_config is not None else proxy_util.config_from_env()
         self._playwright = None
         self._browser: Optional[Browser] = None
         self._context: Optional[BrowserContext] = None
@@ -122,40 +124,12 @@ class PlaywrightLocalBackend(ScraperBackend):
                 f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})"
             )
 
-    @staticmethod
-    def _proxy_from_env() -> dict:
-        """Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled)."""
-        host = os.getenv("BOOKING_PROXY_HOST", "").strip()
-        if not host:
-            return {}
-        return {
-            "host": host,
-            "port": os.getenv("BOOKING_PROXY_PORT", "823").strip(),
-            "username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
-            "password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
-            "country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
-        }
-
     def _proxy_enabled(self) -> bool:
-        return bool(self.proxy_config.get("host") and self.proxy_config.get("username"))
+        return proxy_util.is_enabled(self.proxy_config)
 
     def _new_session_id(self):
         """Pick a fresh sticky-session id — DataImpulse pins one IP per id."""
-        self._session_id = f"hnf{random.randint(100000, 999999)}"
+        self._session_id = proxy_util.new_session_id()
-
-    def _proxy_launch_arg(self) -> Optional[dict]:
-        """Build Playwright's proxy dict, encoding country + sticky session
-        into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID."""
-        if not self._proxy_enabled():
-            return None
-        username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}"
-        if self._session_id:
-            username += f";sessid.{self._session_id}"
-        return {
-            "server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}",
-            "username": username,
-            "password": self.proxy_config["password"],
-        }
 
     async def rotate_session(self):
         """Burn the current proxy IP and warm cache; the next scrape gets a
@@ -198,7 +172,7 @@ class PlaywrightLocalBackend(ScraperBackend):
             # Proxy is set at launch (Chromium binds the sticky session, which
             # lives in the username, at the network layer). Rotating the IP
             # therefore relaunches the browser — see rotate_session().
-            proxy = self._proxy_launch_arg()
+            proxy = proxy_util.playwright_proxy(self.proxy_config, self._session_id)
             if proxy:
                 launch_kwargs['proxy'] = proxy
             self._browser = await self._playwright.chromium.launch(**launch_kwargs)