Extract shared proxy module; make it available to direct scraper; remove dead code

Proxy config, DataImpulse sticky-session username syntax, and Playwright/
httpx proxy builders now live in one place (services/proxy.py) instead of
being duplicated across the Booking.com backend and the /config/proxy test
endpoint. Both scrapers consume it.

- services/proxy.py: load_config/normalize (DB-authoritative, env fallback),
  new_session_id, username, playwright_proxy, httpx_proxy_url
- PlaywrightLocalBackend delegates proxy building to the module
- get_scraper_backend factory uses proxy.load_config (one resolution path)
- test_proxy_config endpoint uses the shared URL builder; httpx proxies= ->
  proxy= (forward-compatible, 0.28-safe)
- Direct booking-engine scraper (httpx) can now route through the same proxy,
  gated by the direct_scraper_use_proxy flag (default off, plumbing ready)

Dead code removed: set_scraper_paused (never called — rotate-on-block
replaced pause-on-block), get_competitor_matrix / get_hotels_list /
update_hotel_tier (endpoints have their own SQL), unused PROXY_KEYS tuple.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/services/booking_scraper.py

@@ -23,6 +23,7 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from database import SyncSessionLocal
+from services import proxy as proxy_util
 from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
 
 logger = logging.getLogger(__name__)
@@ -53,31 +54,9 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
     if backend_type not in ('playwright_local', 'playwright_proxy'):
         logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
 
-    # Proxy configuration is managed in system_config (Settings page). When the
-    # 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
-    # the backend falls back to the BOOKING_PROXY_* environment variables.
-    proxy_rows = db.execute(
-        text("""
-            SELECT config_key, config_value FROM system_config
-            WHERE config_key LIKE 'booking_proxy_%'
-        """)
-    ).fetchall()
-    proxy = {row.config_key: row.config_value for row in proxy_rows}
-
-    if 'booking_proxy_enabled' in proxy:
-        if proxy.get('booking_proxy_enabled') == 'true':
-            return PlaywrightLocalBackend(proxy_config={
-                'host': proxy.get('booking_proxy_host', ''),
-                'port': proxy.get('booking_proxy_port', '823'),
-                'username': proxy.get('booking_proxy_username', ''),
-                'password': proxy.get('booking_proxy_password', ''),
-                'country': proxy.get('booking_proxy_country', 'gb'),
-            })
-        # Explicitly disabled in the DB — direct connection, ignore env.
-        return PlaywrightLocalBackend(proxy_config={})
-
-    # No DB override — let the backend read BOOKING_PROXY_* env vars.
-    return PlaywrightLocalBackend()
+    # Proxy config resolved by the shared module: system_config is authoritative
+    # when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct).
+    return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db))
 
 
 def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
@@ -135,21 +114,6 @@ async def is_scraper_paused(db: Session) -> bool:
     return True
 
 
-async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
-    """Set scraper pause status."""
-    db.execute(
-        text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
-        {'val': 'true' if paused else 'false'}
-    )
-    if paused:
-        pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
-        db.execute(
-            text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
-            {'val': pause_until}
-        )
-    db.commit()
-
-
 def save_hotel(db: Session, hotel: HotelData) -> int:
     """
     Save or update a hotel in the database.
@@ -861,105 +825,3 @@ async def _process_queue_locked(db: Session) -> Dict[str, Any]:
             'dates_failed': len(jobs),
         }
 
-
-def get_competitor_matrix(
-    db: Session,
-    from_date: date,
-    to_date: date,
-    include_market: bool = False
-) -> List[Dict[str, Any]]:
-    """
-    Get rate comparison matrix for competitors.
-
-    Args:
-        db: Database session
-        from_date: Start date
-        to_date: End date
-        include_market: Include market tier hotels
-
-    Returns:
-        List of rate records for matrix display
-    """
-    tier_filter = "h.tier IN ('own', 'competitor')"
-    if include_market:
-        tier_filter = "h.tier IN ('own', 'competitor', 'market')"
-
-    result = db.execute(
-        text(f"""
-            SELECT
-                r.rate_date,
-                h.id AS hotel_id,
-                h.name AS hotel_name,
-                h.tier,
-                h.display_order,
-                h.star_rating,
-                h.review_score,
-                r.availability_status,
-                r.rate_gross,
-                r.room_type,
-                r.breakfast_included,
-                r.free_cancellation,
-                r.no_prepayment,
-                r.rooms_left,
-                r.scraped_at
-            FROM booking_latest_rates r
-            JOIN booking_com_hotels h ON r.hotel_id = h.id
-            WHERE {tier_filter}
-              AND h.is_active = TRUE
-              AND r.rate_date BETWEEN :from_date AND :to_date
-            ORDER BY r.rate_date, h.display_order, h.name
-        """),
-        {'from_date': from_date, 'to_date': to_date}
-    )
-
-    return [dict(row._mapping) for row in result.fetchall()]
-
-
-def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
-    """
-    Get list of discovered hotels.
-
-    Args:
-        db: Database session
-        tier: Filter by tier ('own', 'competitor', 'market') or None for all
-
-    Returns:
-        List of hotel records
-    """
-    where_clause = "WHERE is_active = TRUE"
-    if tier:
-        where_clause += f" AND tier = '{tier}'"
-
-    result = db.execute(
-        text(f"""
-            SELECT
-                id, booking_com_id, name, booking_com_url,
-                star_rating, review_score, review_count,
-                tier, display_order, notes,
-                first_seen_at, last_seen_at
-            FROM booking_com_hotels
-            {where_clause}
-            ORDER BY display_order, name
-        """)
-    )
-
-    return [dict(row._mapping) for row in result.fetchall()]
-
-
-def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
-    """Update a hotel's tier and display order."""
-    if tier not in ('own', 'competitor', 'market'):
-        raise ValueError(f"Invalid tier: {tier}")
-
-    params = {'hotel_id': hotel_id, 'tier': tier}
-    set_clause = "tier = :tier"
-
-    if display_order is not None:
-        set_clause += ", display_order = :order"
-        params['order'] = display_order
-
-    db.execute(
-        text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
-        params
-    )
-    db.commit()