diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 3123e30..99acbd9 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -12,6 +12,7 @@ import logging from database import get_db, SyncSessionLocal from auth import get_current_user +from services import proxy as proxy_util router = APIRouter() logger = logging.getLogger(__name__) @@ -188,12 +189,6 @@ async def set_system_config( # ── Booking.com scraper proxy config ───────────────────────────────────────── -PROXY_KEYS = ( - 'booking_proxy_enabled', 'booking_proxy_host', 'booking_proxy_port', - 'booking_proxy_username', 'booking_proxy_password', 'booking_proxy_country', -) - - class ProxyConfig(BaseModel): enabled: bool = False host: str = '' @@ -265,22 +260,24 @@ async def test_proxy_config( result = await db.execute( text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'") ) - cfg = {row.config_key: row.config_value for row in result.fetchall()} - - host = (cfg.get('booking_proxy_host') or '').strip() - username = (cfg.get('booking_proxy_username') or '').strip() - password = (cfg.get('booking_proxy_password') or '').strip() - if not (host and username and password): + raw = {row.config_key: row.config_value for row in result.fetchall()} + # Test whatever creds are stored, regardless of the enabled toggle, so the + # user can verify before switching the proxy on. + cfg = { + 'host': (raw.get('booking_proxy_host') or '').strip(), + 'port': (raw.get('booking_proxy_port') or '823').strip(), + 'username': (raw.get('booking_proxy_username') or '').strip(), + 'password': (raw.get('booking_proxy_password') or '').strip(), + 'country': (raw.get('booking_proxy_country') or 'gb').strip(), + } + if not (proxy_util.is_enabled(cfg) and cfg['password']): raise HTTPException(status_code=400, detail="Proxy host, username and password must be saved first.") - port = (cfg.get('booking_proxy_port') or '823').strip() - country = (cfg.get('booking_proxy_country') or 'gb').strip() - proxy_user = f"{username}__cr.{country};sessid.hnftest" - proxy_url = f"http://{proxy_user}:{password}@{host}:{port}" + proxy_url = proxy_util.httpx_proxy_url(cfg, proxy_util.new_session_id()) import httpx try: - async with httpx.AsyncClient(proxies=proxy_url, timeout=40.0) as client: + async with httpx.AsyncClient(proxy=proxy_url, timeout=40.0) as client: resp = await client.get("https://ipinfo.io/json") resp.raise_for_status() data = resp.json() diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index b1c2797..8b70ca4 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -23,6 +23,7 @@ from sqlalchemy import text from sqlalchemy.orm import Session from database import SyncSessionLocal +from services import proxy as proxy_util from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus logger = logging.getLogger(__name__) @@ -53,31 +54,9 @@ def get_scraper_backend(db: Session) -> ScraperBackend: if backend_type not in ('playwright_local', 'playwright_proxy'): logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local") - # Proxy configuration is managed in system_config (Settings page). When the - # 'booking_proxy_enabled' key is present the DB is authoritative; otherwise - # the backend falls back to the BOOKING_PROXY_* environment variables. - proxy_rows = db.execute( - text(""" - SELECT config_key, config_value FROM system_config - WHERE config_key LIKE 'booking_proxy_%' - """) - ).fetchall() - proxy = {row.config_key: row.config_value for row in proxy_rows} - - if 'booking_proxy_enabled' in proxy: - if proxy.get('booking_proxy_enabled') == 'true': - return PlaywrightLocalBackend(proxy_config={ - 'host': proxy.get('booking_proxy_host', ''), - 'port': proxy.get('booking_proxy_port', '823'), - 'username': proxy.get('booking_proxy_username', ''), - 'password': proxy.get('booking_proxy_password', ''), - 'country': proxy.get('booking_proxy_country', 'gb'), - }) - # Explicitly disabled in the DB — direct connection, ignore env. - return PlaywrightLocalBackend(proxy_config={}) - - # No DB override — let the backend read BOOKING_PROXY_* env vars. - return PlaywrightLocalBackend() + # Proxy config resolved by the shared module: system_config is authoritative + # when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct). + return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db)) def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: @@ -135,21 +114,6 @@ async def is_scraper_paused(db: Session) -> bool: return True -async def set_scraper_paused(db: Session, paused: bool, hours: int = 2): - """Set scraper pause status.""" - db.execute( - text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"), - {'val': 'true' if paused else 'false'} - ) - if paused: - pause_until = (datetime.now() + timedelta(hours=hours)).isoformat() - db.execute( - text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"), - {'val': pause_until} - ) - db.commit() - - def save_hotel(db: Session, hotel: HotelData) -> int: """ Save or update a hotel in the database. @@ -861,105 +825,3 @@ async def _process_queue_locked(db: Session) -> Dict[str, Any]: 'dates_failed': len(jobs), } - -def get_competitor_matrix( - db: Session, - from_date: date, - to_date: date, - include_market: bool = False -) -> List[Dict[str, Any]]: - """ - Get rate comparison matrix for competitors. - - Args: - db: Database session - from_date: Start date - to_date: End date - include_market: Include market tier hotels - - Returns: - List of rate records for matrix display - """ - tier_filter = "h.tier IN ('own', 'competitor')" - if include_market: - tier_filter = "h.tier IN ('own', 'competitor', 'market')" - - result = db.execute( - text(f""" - SELECT - r.rate_date, - h.id AS hotel_id, - h.name AS hotel_name, - h.tier, - h.display_order, - h.star_rating, - h.review_score, - r.availability_status, - r.rate_gross, - r.room_type, - r.breakfast_included, - r.free_cancellation, - r.no_prepayment, - r.rooms_left, - r.scraped_at - FROM booking_latest_rates r - JOIN booking_com_hotels h ON r.hotel_id = h.id - WHERE {tier_filter} - AND h.is_active = TRUE - AND r.rate_date BETWEEN :from_date AND :to_date - ORDER BY r.rate_date, h.display_order, h.name - """), - {'from_date': from_date, 'to_date': to_date} - ) - - return [dict(row._mapping) for row in result.fetchall()] - - -def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]: - """ - Get list of discovered hotels. - - Args: - db: Database session - tier: Filter by tier ('own', 'competitor', 'market') or None for all - - Returns: - List of hotel records - """ - where_clause = "WHERE is_active = TRUE" - if tier: - where_clause += f" AND tier = '{tier}'" - - result = db.execute( - text(f""" - SELECT - id, booking_com_id, name, booking_com_url, - star_rating, review_score, review_count, - tier, display_order, notes, - first_seen_at, last_seen_at - FROM booking_com_hotels - {where_clause} - ORDER BY display_order, name - """) - ) - - return [dict(row._mapping) for row in result.fetchall()] - - -def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None): - """Update a hotel's tier and display order.""" - if tier not in ('own', 'competitor', 'market'): - raise ValueError(f"Invalid tier: {tier}") - - params = {'hotel_id': hotel_id, 'tier': tier} - set_clause = "tier = :tier" - - if display_order is not None: - set_clause += ", display_order = :order" - params['order'] = display_order - - db.execute( - text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"), - params - ) - db.commit() diff --git a/backend/services/direct_scraper.py b/backend/services/direct_scraper.py index aba16e2..1687c35 100644 --- a/backend/services/direct_scraper.py +++ b/backend/services/direct_scraper.py @@ -11,6 +11,7 @@ import httpx from sqlalchemy import text from database import SyncSessionLocal +from services import proxy as proxy_util from services.direct_profiles import get_profile log = logging.getLogger(__name__) @@ -18,6 +19,17 @@ log = logging.getLogger(__name__) REQUEST_DELAY = 10.0 DISCOVERY_DELAY = 2.0 + +def _direct_proxy_url() -> str | None: + """Proxy URL for direct-engine scraping, or None. Shares the Booking.com + proxy config but is gated by the `direct_scraper_use_proxy` flag (default + off — booking-engine APIs usually don't need it). Ready to switch on.""" + db = SyncSessionLocal() + try: + return proxy_util.direct_httpx_proxy(db) + finally: + db.close() + # Track running discovery scrapes: hotel_id -> status dict _discovery_status: dict[int, dict] = {} @@ -50,7 +62,7 @@ async def run_discovery(hotel_id: int, profile_name: str, params: dict): found_rooms: set[str] = set() found_rates: set[str] = set() - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client: for i, arrival in enumerate(dates): await asyncio.sleep(DISCOVERY_DELAY) try: @@ -127,7 +139,7 @@ def run_scrape(hotel_id: int, profile_name: str, params: dict): async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime): - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client: try: arrival_dates = await profile.fetch_arrival_dates(client, params) except Exception as e: diff --git a/backend/services/proxy.py b/backend/services/proxy.py new file mode 100644 index 0000000..ea8e30a --- /dev/null +++ b/backend/services/proxy.py @@ -0,0 +1,108 @@ +""" +Shared residential-proxy configuration for every scraper. + +Single source of truth for: + - where proxy config lives (system_config `booking_proxy_*` keys, with + BOOKING_PROXY_* env fallback), + - the DataImpulse sticky-session username syntax (LOGIN__cr.;sessid.), + - building proxy settings for both Playwright (launch dict) and httpx (URL). + +The Booking.com browser scraper (Playwright) and the direct booking-engine +scraper (httpx) both consume this, so proxy logic is never duplicated. Direct +scraping is opt-in via the `direct_scraper_use_proxy` config key (default off — +booking-engine APIs generally don't need it), but the plumbing is ready. +""" +import os +import random +from typing import Optional + + +def new_session_id() -> str: + """A fresh sticky-session id. DataImpulse pins one IP per id, so a new id + yields a new residential IP.""" + return f"hnf{random.randint(100000, 999999)}" + + +def config_from_env() -> dict: + """Proxy config from BOOKING_PROXY_* env vars ({} when unset).""" + host = os.getenv("BOOKING_PROXY_HOST", "").strip() + if not host: + return {} + return { + 'host': host, + 'port': os.getenv("BOOKING_PROXY_PORT", "823").strip(), + 'username': os.getenv("BOOKING_PROXY_USERNAME", "").strip(), + 'password': os.getenv("BOOKING_PROXY_PASSWORD", "").strip(), + 'country': os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(), + } + + +def normalize(raw: dict) -> dict: + """Turn raw `booking_proxy_*` system_config values into a proxy config dict. + DB is authoritative when `booking_proxy_enabled` is present; otherwise fall + back to env. Returns {} when disabled or unconfigured.""" + if 'booking_proxy_enabled' in raw: + if raw.get('booking_proxy_enabled') != 'true': + return {} + return { + 'host': (raw.get('booking_proxy_host') or '').strip(), + 'port': (raw.get('booking_proxy_port') or '823').strip(), + 'username': (raw.get('booking_proxy_username') or '').strip(), + 'password': (raw.get('booking_proxy_password') or '').strip(), + 'country': (raw.get('booking_proxy_country') or 'gb').strip(), + } + return config_from_env() + + +def load_config(db) -> dict: + """Resolve proxy config using a sync SQLAlchemy session. Returns {} when + disabled/unconfigured. (Async callers should fetch the rows themselves and + pass them through normalize().)""" + from sqlalchemy import text + rows = db.execute( + text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'") + ).fetchall() + return normalize({r.config_key: r.config_value for r in rows}) + + +def is_enabled(cfg: dict) -> bool: + return bool(cfg.get('host') and cfg.get('username')) + + +def username(cfg: dict, session_id: Optional[str] = None) -> str: + """DataImpulse username: LOGIN__cr.[;sessid.].""" + user = f"{cfg['username']}__cr.{cfg.get('country', 'gb')}" + if session_id: + user += f";sessid.{session_id}" + return user + + +def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]: + """Proxy dict for chromium.launch(proxy=...). None when disabled.""" + if not is_enabled(cfg): + return None + return { + 'server': f"http://{cfg['host']}:{cfg['port']}", + 'username': username(cfg, session_id), + 'password': cfg['password'], + } + + +def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]: + """Proxy URL for httpx.AsyncClient(proxies=...). None when disabled.""" + if not is_enabled(cfg): + return None + return f"http://{username(cfg, session_id)}:{cfg['password']}@{cfg['host']}:{cfg['port']}" + + +def direct_httpx_proxy(db) -> Optional[str]: + """Proxy URL for the direct booking-engine scraper — only when the proxy is + configured AND `direct_scraper_use_proxy` is explicitly enabled (default off). + Uses a fresh session id per call so direct runs spread across IPs.""" + from sqlalchemy import text + flag = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'direct_scraper_use_proxy'") + ).fetchone() + if not flag or flag.config_value != 'true': + return None + return httpx_proxy_url(load_config(db), new_session_id()) diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index faeb28b..75304c2 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -7,7 +7,6 @@ No proxy - direct connection. Suitable for low-volume scraping. import asyncio import logging -import os import random import re from datetime import date @@ -15,6 +14,8 @@ from decimal import Decimal, InvalidOperation from typing import List, Optional from urllib.parse import urlencode, urlparse, parse_qs +from services import proxy as proxy_util + def location_params_from_url(url: str) -> dict: """Pull the stable destination params (ss, dest_id, dest_type) out of a @@ -109,8 +110,9 @@ class PlaywrightLocalBackend(ScraperBackend): Args: proxy_config: Optional proxy configuration. Falls back to the BOOKING_PROXY_* environment variables when not supplied. + See services/proxy.py for the shared config/build helpers. """ - self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env() + self.proxy_config = proxy_config if proxy_config is not None else proxy_util.config_from_env() self._playwright = None self._browser: Optional[Browser] = None self._context: Optional[BrowserContext] = None @@ -122,40 +124,12 @@ class PlaywrightLocalBackend(ScraperBackend): f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})" ) - @staticmethod - def _proxy_from_env() -> dict: - """Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled).""" - host = os.getenv("BOOKING_PROXY_HOST", "").strip() - if not host: - return {} - return { - "host": host, - "port": os.getenv("BOOKING_PROXY_PORT", "823").strip(), - "username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(), - "password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(), - "country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(), - } - def _proxy_enabled(self) -> bool: - return bool(self.proxy_config.get("host") and self.proxy_config.get("username")) + return proxy_util.is_enabled(self.proxy_config) def _new_session_id(self): """Pick a fresh sticky-session id — DataImpulse pins one IP per id.""" - self._session_id = f"hnf{random.randint(100000, 999999)}" - - def _proxy_launch_arg(self) -> Optional[dict]: - """Build Playwright's proxy dict, encoding country + sticky session - into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID.""" - if not self._proxy_enabled(): - return None - username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}" - if self._session_id: - username += f";sessid.{self._session_id}" - return { - "server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}", - "username": username, - "password": self.proxy_config["password"], - } + self._session_id = proxy_util.new_session_id() async def rotate_session(self): """Burn the current proxy IP and warm cache; the next scrape gets a @@ -198,7 +172,7 @@ class PlaywrightLocalBackend(ScraperBackend): # Proxy is set at launch (Chromium binds the sticky session, which # lives in the username, at the network layer). Rotating the IP # therefore relaunches the browser — see rotate_session(). - proxy = self._proxy_launch_arg() + proxy = proxy_util.playwright_proxy(self.proxy_config, self._session_id) if proxy: launch_kwargs['proxy'] = proxy self._browser = await self._playwright.chromium.launch(**launch_kwargs)