diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 915cd00..420b2f6 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -149,9 +149,15 @@ async def get_system_config( db: AsyncSession = Depends(get_db), current_user: dict = Depends(get_current_user) ): - """Return all system_config rows as a flat dict.""" + """Return all system_config rows as a flat dict (secrets masked).""" result = await db.execute(text("SELECT config_key, config_value FROM system_config")) - return {row.config_key: row.config_value for row in result.fetchall()} + out = {} + for row in result.fetchall(): + if row.config_value and ('password' in row.config_key or 'secret' in row.config_key): + out[row.config_key] = '********' + else: + out[row.config_key] = row.config_value + return out class SystemConfigUpdate(BaseModel): @@ -178,6 +184,117 @@ async def set_system_config( return {"status": "success", "key": payload.key} +# ── Booking.com scraper proxy config ───────────────────────────────────────── + +PROXY_KEYS = ( + 'booking_proxy_enabled', 'booking_proxy_host', 'booking_proxy_port', + 'booking_proxy_username', 'booking_proxy_password', 'booking_proxy_country', +) + + +class ProxyConfig(BaseModel): + enabled: bool = False + host: str = '' + port: str = '823' + username: str = '' + password: Optional[str] = None # None/'' => keep the stored password + country: str = 'gb' + + +@router.get("/config/proxy") +async def get_proxy_config( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Return the scraper proxy config. Password is never returned — only + a `password_set` flag indicating whether one is stored.""" + result = await db.execute( + text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'") + ) + cfg = {row.config_key: row.config_value for row in result.fetchall()} + return { + 'enabled': cfg.get('booking_proxy_enabled') == 'true', + 'host': cfg.get('booking_proxy_host', ''), + 'port': cfg.get('booking_proxy_port', '823'), + 'username': cfg.get('booking_proxy_username', ''), + 'country': cfg.get('booking_proxy_country', 'gb'), + 'password_set': bool(cfg.get('booking_proxy_password')), + } + + +@router.post("/config/proxy") +async def set_proxy_config( + payload: ProxyConfig, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Upsert the scraper proxy config. A blank password leaves the stored + one untouched, so the UI never has to round-trip the secret.""" + values = { + 'booking_proxy_enabled': 'true' if payload.enabled else 'false', + 'booking_proxy_host': payload.host.strip(), + 'booking_proxy_port': (payload.port or '823').strip(), + 'booking_proxy_username': payload.username.strip(), + 'booking_proxy_country': (payload.country or 'gb').strip(), + } + if payload.password: # only overwrite when a new value is provided + values['booking_proxy_password'] = payload.password.strip() + + for key, value in values.items(): + await db.execute( + text(""" + INSERT INTO system_config (config_key, config_value) + VALUES (:key, :value) + ON CONFLICT (config_key) DO UPDATE SET config_value = EXCLUDED.config_value + """), + {'key': key, 'value': value} + ) + await db.commit() + return {"status": "success"} + + +@router.post("/config/proxy/test") +async def test_proxy_config( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Make a live request through the configured proxy and report the exit + IP and country, so the user can confirm credentials + geo before scraping.""" + result = await db.execute( + text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'") + ) + cfg = {row.config_key: row.config_value for row in result.fetchall()} + + host = (cfg.get('booking_proxy_host') or '').strip() + username = (cfg.get('booking_proxy_username') or '').strip() + password = (cfg.get('booking_proxy_password') or '').strip() + if not (host and username and password): + raise HTTPException(status_code=400, detail="Proxy host, username and password must be saved first.") + + port = (cfg.get('booking_proxy_port') or '823').strip() + country = (cfg.get('booking_proxy_country') or 'gb').strip() + proxy_user = f"{username}__cr.{country};sessid.hnftest" + proxy_url = f"http://{proxy_user}:{password}@{host}:{port}" + + import httpx + try: + async with httpx.AsyncClient(proxies=proxy_url, timeout=40.0) as client: + resp = await client.get("https://ipinfo.io/json") + resp.raise_for_status() + data = resp.json() + except Exception as e: + logger.warning(f"Proxy test failed: {e}") + raise HTTPException(status_code=502, detail=f"Proxy test failed: {e}") + + return { + 'ok': True, + 'ip': data.get('ip'), + 'country': data.get('country'), + 'city': data.get('city'), + 'org': data.get('org'), + } + + @router.post("/config/location") async def set_location_config( config: LocationConfigRequest, diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 992db1b..826d897 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -44,27 +44,38 @@ def get_scraper_backend(db: Session) -> ScraperBackend: backend_type = result.config_value if result and result.config_value else 'playwright_local' - if backend_type == 'playwright_local': - return PlaywrightLocalBackend() - - elif backend_type == 'playwright_proxy': - # Get proxy config - proxy_result = db.execute( - text(""" - SELECT config_key, config_value FROM system_config - WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password') - """) - ) - proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()} - return PlaywrightLocalBackend(proxy_config=proxy_config) - - elif backend_type == 'apify': + if backend_type == 'apify': # Future: Apify backend raise NotImplementedError("Apify backend not yet implemented") - else: + if backend_type not in ('playwright_local', 'playwright_proxy'): logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local") - return PlaywrightLocalBackend() + + # Proxy configuration is managed in system_config (Settings page). When the + # 'booking_proxy_enabled' key is present the DB is authoritative; otherwise + # the backend falls back to the BOOKING_PROXY_* environment variables. + proxy_rows = db.execute( + text(""" + SELECT config_key, config_value FROM system_config + WHERE config_key LIKE 'booking_proxy_%' + """) + ).fetchall() + proxy = {row.config_key: row.config_value for row in proxy_rows} + + if 'booking_proxy_enabled' in proxy: + if proxy.get('booking_proxy_enabled') == 'true': + return PlaywrightLocalBackend(proxy_config={ + 'host': proxy.get('booking_proxy_host', ''), + 'port': proxy.get('booking_proxy_port', '823'), + 'username': proxy.get('booking_proxy_username', ''), + 'password': proxy.get('booking_proxy_password', ''), + 'country': proxy.get('booking_proxy_country', 'gb'), + }) + # Explicitly disabled in the DB — direct connection, ignore env. + return PlaywrightLocalBackend(proxy_config={}) + + # No DB override — let the backend read BOOKING_PROXY_* env vars. + return PlaywrightLocalBackend() def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index e31a070..0f2fd9b 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -1,11 +1,12 @@ -import { useState } from 'react' +import { useState, useEffect } from 'react' import { useParams, useNavigate } from 'react-router-dom' import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' -import { Save, RefreshCw, Database, Clock, BedDouble, ChevronUp, ChevronDown } from 'lucide-react' +import { Save, RefreshCw, Database, Clock, BedDouble, ChevronUp, ChevronDown, Network, ShieldCheck } from 'lucide-react' import api from '../api' const TABS = [ { id: 'newbook', label: 'Newbook Sync' }, + { id: 'proxy', label: 'Scraper Proxy' }, { id: 'system', label: 'System' }, ] @@ -76,6 +77,10 @@ export default function Settings() { /> )} + {activeTab === 'proxy' && ( + + )} + {activeTab === 'system' && ( )} @@ -83,6 +88,170 @@ export default function Settings() { ) } +// ─── Scraper Proxy Tab ──────────────────────────────────────────────────────── + +interface ProxyConfigData { + enabled: boolean + host: string + port: string + username: string + country: string + password_set: boolean +} + +interface ProxyTestResult { + ok: boolean + ip: string + country: string + city: string + org: string +} + +function ProxyTab() { + const qc = useQueryClient() + + const { data: cfg, isLoading } = useQuery({ + queryKey: ['proxy-config'], + queryFn: () => api.get('/competitors/config/proxy').then(r => r.data), + }) + + const [enabled, setEnabled] = useState(false) + const [host, setHost] = useState('') + const [port, setPort] = useState('823') + const [username, setUsername] = useState('') + const [country, setCountry] = useState('gb') + const [password, setPassword] = useState('') + const [testResult, setTestResult] = useState(null) + const [testError, setTestError] = useState(null) + + // Seed form once the stored config arrives (password stays blank by design) + useEffect(() => { + if (cfg) { + setEnabled(cfg.enabled) + setHost(cfg.host) + setPort(cfg.port) + setUsername(cfg.username) + setCountry(cfg.country) + } + }, [cfg]) + + const save = useMutation({ + mutationFn: () => + api.post('/competitors/config/proxy', { + enabled, host, port, username, country, + // Only send a password when the user typed a new one + password: password || undefined, + }), + onSuccess: () => { + setPassword('') + qc.invalidateQueries({ queryKey: ['proxy-config'] }) + qc.invalidateQueries({ queryKey: ['system-config'] }) + }, + }) + + const runTest = useMutation({ + mutationFn: () => api.post('/competitors/config/proxy/test').then(r => r.data as ProxyTestResult), + onMutate: () => { setTestResult(null); setTestError(null) }, + onSuccess: (data) => setTestResult(data), + onError: (err: any) => setTestError(err?.response?.data?.detail || 'Test failed'), + }) + + if (isLoading) { + return
Loading…
+ } + + const labelStyle: React.CSSProperties = { + fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 6, + } + + return ( +
+
+
+ + + Residential Proxy + + + {enabled ? 'Enabled' : 'Disabled'} + +
+
+

+ Routes the Booking.com scraper through a residential proxy (e.g. DataImpulse). + One sticky IP is held per scrape session and rotated automatically if a page is + blocked. Leave disabled to scrape from this server's own IP. +

+ + + +
+
+ + setHost(e.target.value)} style={{ width: '100%' }} /> +
+
+ + setPort(e.target.value)} style={{ width: '100%' }} /> +
+
+ +
+ + setUsername(e.target.value)} style={{ width: '100%' }} /> +
+ +
+
+ + setPassword(e.target.value)} style={{ width: '100%' }} /> +
+
+ + setCountry(e.target.value.toLowerCase())} style={{ width: '100%' }} /> +
+
+ +
+ + +
+ + {save.isError && ( +

+ {(save.error as any)?.response?.data?.detail || 'Save failed'} +

+ )} + {testResult && ( +
+ ✓ Connected — exit IP {testResult.ip} ({testResult.country} + {testResult.city ? `, ${testResult.city}` : ''}){testResult.org ? ` · ${testResult.org}` : ''} +
+ )} + {testError && ( +

{testError}

+ )} +
+
+
+ ) +} + // ─── Newbook Sync Tab ───────────────────────────────────────────────────────── interface NewbookTabProps { @@ -321,6 +490,9 @@ function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; is 'booking_scraper_paused', 'booking_scraper_backend', 'booking_scraper_daily_time', + 'booking_proxy_enabled', + 'booking_proxy_host', + 'booking_proxy_country', 'sync_newbook_current_rates_enabled', 'sync_newbook_current_rates_time', ]