Manage scraper proxy from the Settings page
- New "Scraper Proxy" tab: enable toggle, host/port/username/password/ country, Save, and a Test Connection button that reports the live exit IP + country through the proxy - Backend proxy config now lives in system_config (DB authoritative when booking_proxy_enabled is set; BOOKING_PROXY_* env vars are the fallback) - Dedicated /config/proxy GET/POST/test endpoints; password is write-only (never returned, blank keeps the stored value) and masked in /config/system - Surface proxy status keys in the read-only System tab Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
11b995739c
commit
270d8293d1
3 changed files with 321 additions and 21 deletions
|
|
@ -149,9 +149,15 @@ async def get_system_config(
|
|||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Return all system_config rows as a flat dict."""
|
||||
"""Return all system_config rows as a flat dict (secrets masked)."""
|
||||
result = await db.execute(text("SELECT config_key, config_value FROM system_config"))
|
||||
return {row.config_key: row.config_value for row in result.fetchall()}
|
||||
out = {}
|
||||
for row in result.fetchall():
|
||||
if row.config_value and ('password' in row.config_key or 'secret' in row.config_key):
|
||||
out[row.config_key] = '********'
|
||||
else:
|
||||
out[row.config_key] = row.config_value
|
||||
return out
|
||||
|
||||
|
||||
class SystemConfigUpdate(BaseModel):
|
||||
|
|
@ -178,6 +184,117 @@ async def set_system_config(
|
|||
return {"status": "success", "key": payload.key}
|
||||
|
||||
|
||||
# ── Booking.com scraper proxy config ─────────────────────────────────────────
|
||||
|
||||
PROXY_KEYS = (
|
||||
'booking_proxy_enabled', 'booking_proxy_host', 'booking_proxy_port',
|
||||
'booking_proxy_username', 'booking_proxy_password', 'booking_proxy_country',
|
||||
)
|
||||
|
||||
|
||||
class ProxyConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
host: str = ''
|
||||
port: str = '823'
|
||||
username: str = ''
|
||||
password: Optional[str] = None # None/'' => keep the stored password
|
||||
country: str = 'gb'
|
||||
|
||||
|
||||
@router.get("/config/proxy")
|
||||
async def get_proxy_config(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Return the scraper proxy config. Password is never returned — only
|
||||
a `password_set` flag indicating whether one is stored."""
|
||||
result = await db.execute(
|
||||
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
|
||||
)
|
||||
cfg = {row.config_key: row.config_value for row in result.fetchall()}
|
||||
return {
|
||||
'enabled': cfg.get('booking_proxy_enabled') == 'true',
|
||||
'host': cfg.get('booking_proxy_host', ''),
|
||||
'port': cfg.get('booking_proxy_port', '823'),
|
||||
'username': cfg.get('booking_proxy_username', ''),
|
||||
'country': cfg.get('booking_proxy_country', 'gb'),
|
||||
'password_set': bool(cfg.get('booking_proxy_password')),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/config/proxy")
|
||||
async def set_proxy_config(
|
||||
payload: ProxyConfig,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Upsert the scraper proxy config. A blank password leaves the stored
|
||||
one untouched, so the UI never has to round-trip the secret."""
|
||||
values = {
|
||||
'booking_proxy_enabled': 'true' if payload.enabled else 'false',
|
||||
'booking_proxy_host': payload.host.strip(),
|
||||
'booking_proxy_port': (payload.port or '823').strip(),
|
||||
'booking_proxy_username': payload.username.strip(),
|
||||
'booking_proxy_country': (payload.country or 'gb').strip(),
|
||||
}
|
||||
if payload.password: # only overwrite when a new value is provided
|
||||
values['booking_proxy_password'] = payload.password.strip()
|
||||
|
||||
for key, value in values.items():
|
||||
await db.execute(
|
||||
text("""
|
||||
INSERT INTO system_config (config_key, config_value)
|
||||
VALUES (:key, :value)
|
||||
ON CONFLICT (config_key) DO UPDATE SET config_value = EXCLUDED.config_value
|
||||
"""),
|
||||
{'key': key, 'value': value}
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "success"}
|
||||
|
||||
|
||||
@router.post("/config/proxy/test")
|
||||
async def test_proxy_config(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Make a live request through the configured proxy and report the exit
|
||||
IP and country, so the user can confirm credentials + geo before scraping."""
|
||||
result = await db.execute(
|
||||
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
|
||||
)
|
||||
cfg = {row.config_key: row.config_value for row in result.fetchall()}
|
||||
|
||||
host = (cfg.get('booking_proxy_host') or '').strip()
|
||||
username = (cfg.get('booking_proxy_username') or '').strip()
|
||||
password = (cfg.get('booking_proxy_password') or '').strip()
|
||||
if not (host and username and password):
|
||||
raise HTTPException(status_code=400, detail="Proxy host, username and password must be saved first.")
|
||||
|
||||
port = (cfg.get('booking_proxy_port') or '823').strip()
|
||||
country = (cfg.get('booking_proxy_country') or 'gb').strip()
|
||||
proxy_user = f"{username}__cr.{country};sessid.hnftest"
|
||||
proxy_url = f"http://{proxy_user}:{password}@{host}:{port}"
|
||||
|
||||
import httpx
|
||||
try:
|
||||
async with httpx.AsyncClient(proxies=proxy_url, timeout=40.0) as client:
|
||||
resp = await client.get("https://ipinfo.io/json")
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
logger.warning(f"Proxy test failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Proxy test failed: {e}")
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'ip': data.get('ip'),
|
||||
'country': data.get('country'),
|
||||
'city': data.get('city'),
|
||||
'org': data.get('org'),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/config/location")
|
||||
async def set_location_config(
|
||||
config: LocationConfigRequest,
|
||||
|
|
|
|||
|
|
@ -44,27 +44,38 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
|
|||
|
||||
backend_type = result.config_value if result and result.config_value else 'playwright_local'
|
||||
|
||||
if backend_type == 'playwright_local':
|
||||
return PlaywrightLocalBackend()
|
||||
|
||||
elif backend_type == 'playwright_proxy':
|
||||
# Get proxy config
|
||||
proxy_result = db.execute(
|
||||
text("""
|
||||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password')
|
||||
""")
|
||||
)
|
||||
proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()}
|
||||
return PlaywrightLocalBackend(proxy_config=proxy_config)
|
||||
|
||||
elif backend_type == 'apify':
|
||||
if backend_type == 'apify':
|
||||
# Future: Apify backend
|
||||
raise NotImplementedError("Apify backend not yet implemented")
|
||||
|
||||
else:
|
||||
if backend_type not in ('playwright_local', 'playwright_proxy'):
|
||||
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
|
||||
return PlaywrightLocalBackend()
|
||||
|
||||
# Proxy configuration is managed in system_config (Settings page). When the
|
||||
# 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
|
||||
# the backend falls back to the BOOKING_PROXY_* environment variables.
|
||||
proxy_rows = db.execute(
|
||||
text("""
|
||||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key LIKE 'booking_proxy_%'
|
||||
""")
|
||||
).fetchall()
|
||||
proxy = {row.config_key: row.config_value for row in proxy_rows}
|
||||
|
||||
if 'booking_proxy_enabled' in proxy:
|
||||
if proxy.get('booking_proxy_enabled') == 'true':
|
||||
return PlaywrightLocalBackend(proxy_config={
|
||||
'host': proxy.get('booking_proxy_host', ''),
|
||||
'port': proxy.get('booking_proxy_port', '823'),
|
||||
'username': proxy.get('booking_proxy_username', ''),
|
||||
'password': proxy.get('booking_proxy_password', ''),
|
||||
'country': proxy.get('booking_proxy_country', 'gb'),
|
||||
})
|
||||
# Explicitly disabled in the DB — direct connection, ignore env.
|
||||
return PlaywrightLocalBackend(proxy_config={})
|
||||
|
||||
# No DB override — let the backend read BOOKING_PROXY_* env vars.
|
||||
return PlaywrightLocalBackend()
|
||||
|
||||
|
||||
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import { useState } from 'react'
|
||||
import { useState, useEffect } from 'react'
|
||||
import { useParams, useNavigate } from 'react-router-dom'
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { Save, RefreshCw, Database, Clock, BedDouble, ChevronUp, ChevronDown } from 'lucide-react'
|
||||
import { Save, RefreshCw, Database, Clock, BedDouble, ChevronUp, ChevronDown, Network, ShieldCheck } from 'lucide-react'
|
||||
import api from '../api'
|
||||
|
||||
const TABS = [
|
||||
{ id: 'newbook', label: 'Newbook Sync' },
|
||||
{ id: 'proxy', label: 'Scraper Proxy' },
|
||||
{ id: 'system', label: 'System' },
|
||||
]
|
||||
|
||||
|
|
@ -76,6 +77,10 @@ export default function Settings() {
|
|||
/>
|
||||
)}
|
||||
|
||||
{activeTab === 'proxy' && (
|
||||
<ProxyTab />
|
||||
)}
|
||||
|
||||
{activeTab === 'system' && (
|
||||
<SystemTab config={config} isLoading={isLoading} />
|
||||
)}
|
||||
|
|
@ -83,6 +88,170 @@ export default function Settings() {
|
|||
)
|
||||
}
|
||||
|
||||
// ─── Scraper Proxy Tab ────────────────────────────────────────────────────────
|
||||
|
||||
interface ProxyConfigData {
|
||||
enabled: boolean
|
||||
host: string
|
||||
port: string
|
||||
username: string
|
||||
country: string
|
||||
password_set: boolean
|
||||
}
|
||||
|
||||
interface ProxyTestResult {
|
||||
ok: boolean
|
||||
ip: string
|
||||
country: string
|
||||
city: string
|
||||
org: string
|
||||
}
|
||||
|
||||
function ProxyTab() {
|
||||
const qc = useQueryClient()
|
||||
|
||||
const { data: cfg, isLoading } = useQuery<ProxyConfigData>({
|
||||
queryKey: ['proxy-config'],
|
||||
queryFn: () => api.get('/competitors/config/proxy').then(r => r.data),
|
||||
})
|
||||
|
||||
const [enabled, setEnabled] = useState(false)
|
||||
const [host, setHost] = useState('')
|
||||
const [port, setPort] = useState('823')
|
||||
const [username, setUsername] = useState('')
|
||||
const [country, setCountry] = useState('gb')
|
||||
const [password, setPassword] = useState('')
|
||||
const [testResult, setTestResult] = useState<ProxyTestResult | null>(null)
|
||||
const [testError, setTestError] = useState<string | null>(null)
|
||||
|
||||
// Seed form once the stored config arrives (password stays blank by design)
|
||||
useEffect(() => {
|
||||
if (cfg) {
|
||||
setEnabled(cfg.enabled)
|
||||
setHost(cfg.host)
|
||||
setPort(cfg.port)
|
||||
setUsername(cfg.username)
|
||||
setCountry(cfg.country)
|
||||
}
|
||||
}, [cfg])
|
||||
|
||||
const save = useMutation({
|
||||
mutationFn: () =>
|
||||
api.post('/competitors/config/proxy', {
|
||||
enabled, host, port, username, country,
|
||||
// Only send a password when the user typed a new one
|
||||
password: password || undefined,
|
||||
}),
|
||||
onSuccess: () => {
|
||||
setPassword('')
|
||||
qc.invalidateQueries({ queryKey: ['proxy-config'] })
|
||||
qc.invalidateQueries({ queryKey: ['system-config'] })
|
||||
},
|
||||
})
|
||||
|
||||
const runTest = useMutation({
|
||||
mutationFn: () => api.post('/competitors/config/proxy/test').then(r => r.data as ProxyTestResult),
|
||||
onMutate: () => { setTestResult(null); setTestError(null) },
|
||||
onSuccess: (data) => setTestResult(data),
|
||||
onError: (err: any) => setTestError(err?.response?.data?.detail || 'Test failed'),
|
||||
})
|
||||
|
||||
if (isLoading) {
|
||||
return <div className="loading-state"><div className="spinner" /> Loading…</div>
|
||||
}
|
||||
|
||||
const labelStyle: React.CSSProperties = {
|
||||
fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 6,
|
||||
}
|
||||
|
||||
return (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 20, maxWidth: 600 }}>
|
||||
<div className="card">
|
||||
<div className="card-header">
|
||||
<span style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<Network size={16} strokeWidth={1.75} />
|
||||
Residential Proxy
|
||||
</span>
|
||||
<span className={`badge ${enabled ? 'badge-success' : 'badge-neutral'}`}>
|
||||
{enabled ? 'Enabled' : 'Disabled'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
|
||||
<p style={{ fontSize: 13, color: 'var(--text-mid)', margin: 0 }}>
|
||||
Routes the Booking.com scraper through a residential proxy (e.g. DataImpulse).
|
||||
One sticky IP is held per scrape session and rotated automatically if a page is
|
||||
blocked. Leave disabled to scrape from this server's own IP.
|
||||
</p>
|
||||
|
||||
<label style={{ display: 'flex', alignItems: 'center', gap: 8, fontSize: 13 }}>
|
||||
<input type="checkbox" checked={enabled} onChange={e => setEnabled(e.target.checked)} />
|
||||
Enable proxy for scraping
|
||||
</label>
|
||||
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '2fr 1fr', gap: 12 }}>
|
||||
<div>
|
||||
<label style={labelStyle}>Proxy host</label>
|
||||
<input type="text" placeholder="gw.dataimpulse.com" value={host}
|
||||
onChange={e => setHost(e.target.value)} style={{ width: '100%' }} />
|
||||
</div>
|
||||
<div>
|
||||
<label style={labelStyle}>Port</label>
|
||||
<input type="text" placeholder="823" value={port}
|
||||
onChange={e => setPort(e.target.value)} style={{ width: '100%' }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label style={labelStyle}>Username / login</label>
|
||||
<input type="text" placeholder="account login" value={username}
|
||||
onChange={e => setUsername(e.target.value)} style={{ width: '100%' }} />
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '2fr 1fr', gap: 12 }}>
|
||||
<div>
|
||||
<label style={labelStyle}>Password</label>
|
||||
<input type="password" placeholder={cfg?.password_set ? '•••••••• (unchanged)' : 'not set'}
|
||||
value={password} onChange={e => setPassword(e.target.value)} style={{ width: '100%' }} />
|
||||
</div>
|
||||
<div>
|
||||
<label style={labelStyle}>Country</label>
|
||||
<input type="text" placeholder="gb" value={country}
|
||||
onChange={e => setCountry(e.target.value.toLowerCase())} style={{ width: '100%' }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', gap: 12, alignItems: 'center' }}>
|
||||
<button className="btn btn-primary" onClick={() => save.mutate()} disabled={save.isPending}>
|
||||
<Save size={14} strokeWidth={1.75} />
|
||||
{save.isPending ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
<button className="btn btn-outline" onClick={() => runTest.mutate()} disabled={runTest.isPending}>
|
||||
<ShieldCheck size={14} strokeWidth={1.75} />
|
||||
{runTest.isPending ? 'Testing…' : 'Test Connection'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{save.isError && (
|
||||
<p style={{ fontSize: 13, color: 'var(--danger)', margin: 0 }}>
|
||||
{(save.error as any)?.response?.data?.detail || 'Save failed'}
|
||||
</p>
|
||||
)}
|
||||
{testResult && (
|
||||
<div style={{ fontSize: 13, color: 'var(--success)', background: 'var(--bg-subtle, #f0fdf4)',
|
||||
padding: '10px 12px', borderRadius: 6, border: '1px solid var(--success)' }}>
|
||||
✓ Connected — exit IP <strong>{testResult.ip}</strong> ({testResult.country}
|
||||
{testResult.city ? `, ${testResult.city}` : ''}){testResult.org ? ` · ${testResult.org}` : ''}
|
||||
</div>
|
||||
)}
|
||||
{testError && (
|
||||
<p style={{ fontSize: 13, color: 'var(--danger)', margin: 0 }}>{testError}</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ─── Newbook Sync Tab ─────────────────────────────────────────────────────────
|
||||
|
||||
interface NewbookTabProps {
|
||||
|
|
@ -321,6 +490,9 @@ function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; is
|
|||
'booking_scraper_paused',
|
||||
'booking_scraper_backend',
|
||||
'booking_scraper_daily_time',
|
||||
'booking_proxy_enabled',
|
||||
'booking_proxy_host',
|
||||
'booking_proxy_country',
|
||||
'sync_newbook_current_rates_enabled',
|
||||
'sync_newbook_current_rates_time',
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue