Manage scraper proxy from the Settings page
- New "Scraper Proxy" tab: enable toggle, host/port/username/password/ country, Save, and a Test Connection button that reports the live exit IP + country through the proxy - Backend proxy config now lives in system_config (DB authoritative when booking_proxy_enabled is set; BOOKING_PROXY_* env vars are the fallback) - Dedicated /config/proxy GET/POST/test endpoints; password is write-only (never returned, blank keeps the stored value) and masked in /config/system - Surface proxy status keys in the read-only System tab Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
11b995739c
commit
270d8293d1
3 changed files with 321 additions and 21 deletions
|
|
@ -149,9 +149,15 @@ async def get_system_config(
|
|||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Return all system_config rows as a flat dict."""
|
||||
"""Return all system_config rows as a flat dict (secrets masked)."""
|
||||
result = await db.execute(text("SELECT config_key, config_value FROM system_config"))
|
||||
return {row.config_key: row.config_value for row in result.fetchall()}
|
||||
out = {}
|
||||
for row in result.fetchall():
|
||||
if row.config_value and ('password' in row.config_key or 'secret' in row.config_key):
|
||||
out[row.config_key] = '********'
|
||||
else:
|
||||
out[row.config_key] = row.config_value
|
||||
return out
|
||||
|
||||
|
||||
class SystemConfigUpdate(BaseModel):
|
||||
|
|
@ -178,6 +184,117 @@ async def set_system_config(
|
|||
return {"status": "success", "key": payload.key}
|
||||
|
||||
|
||||
# ── Booking.com scraper proxy config ─────────────────────────────────────────
|
||||
|
||||
PROXY_KEYS = (
|
||||
'booking_proxy_enabled', 'booking_proxy_host', 'booking_proxy_port',
|
||||
'booking_proxy_username', 'booking_proxy_password', 'booking_proxy_country',
|
||||
)
|
||||
|
||||
|
||||
class ProxyConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
host: str = ''
|
||||
port: str = '823'
|
||||
username: str = ''
|
||||
password: Optional[str] = None # None/'' => keep the stored password
|
||||
country: str = 'gb'
|
||||
|
||||
|
||||
@router.get("/config/proxy")
|
||||
async def get_proxy_config(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Return the scraper proxy config. Password is never returned — only
|
||||
a `password_set` flag indicating whether one is stored."""
|
||||
result = await db.execute(
|
||||
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
|
||||
)
|
||||
cfg = {row.config_key: row.config_value for row in result.fetchall()}
|
||||
return {
|
||||
'enabled': cfg.get('booking_proxy_enabled') == 'true',
|
||||
'host': cfg.get('booking_proxy_host', ''),
|
||||
'port': cfg.get('booking_proxy_port', '823'),
|
||||
'username': cfg.get('booking_proxy_username', ''),
|
||||
'country': cfg.get('booking_proxy_country', 'gb'),
|
||||
'password_set': bool(cfg.get('booking_proxy_password')),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/config/proxy")
|
||||
async def set_proxy_config(
|
||||
payload: ProxyConfig,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Upsert the scraper proxy config. A blank password leaves the stored
|
||||
one untouched, so the UI never has to round-trip the secret."""
|
||||
values = {
|
||||
'booking_proxy_enabled': 'true' if payload.enabled else 'false',
|
||||
'booking_proxy_host': payload.host.strip(),
|
||||
'booking_proxy_port': (payload.port or '823').strip(),
|
||||
'booking_proxy_username': payload.username.strip(),
|
||||
'booking_proxy_country': (payload.country or 'gb').strip(),
|
||||
}
|
||||
if payload.password: # only overwrite when a new value is provided
|
||||
values['booking_proxy_password'] = payload.password.strip()
|
||||
|
||||
for key, value in values.items():
|
||||
await db.execute(
|
||||
text("""
|
||||
INSERT INTO system_config (config_key, config_value)
|
||||
VALUES (:key, :value)
|
||||
ON CONFLICT (config_key) DO UPDATE SET config_value = EXCLUDED.config_value
|
||||
"""),
|
||||
{'key': key, 'value': value}
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "success"}
|
||||
|
||||
|
||||
@router.post("/config/proxy/test")
|
||||
async def test_proxy_config(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Make a live request through the configured proxy and report the exit
|
||||
IP and country, so the user can confirm credentials + geo before scraping."""
|
||||
result = await db.execute(
|
||||
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
|
||||
)
|
||||
cfg = {row.config_key: row.config_value for row in result.fetchall()}
|
||||
|
||||
host = (cfg.get('booking_proxy_host') or '').strip()
|
||||
username = (cfg.get('booking_proxy_username') or '').strip()
|
||||
password = (cfg.get('booking_proxy_password') or '').strip()
|
||||
if not (host and username and password):
|
||||
raise HTTPException(status_code=400, detail="Proxy host, username and password must be saved first.")
|
||||
|
||||
port = (cfg.get('booking_proxy_port') or '823').strip()
|
||||
country = (cfg.get('booking_proxy_country') or 'gb').strip()
|
||||
proxy_user = f"{username}__cr.{country};sessid.hnftest"
|
||||
proxy_url = f"http://{proxy_user}:{password}@{host}:{port}"
|
||||
|
||||
import httpx
|
||||
try:
|
||||
async with httpx.AsyncClient(proxies=proxy_url, timeout=40.0) as client:
|
||||
resp = await client.get("https://ipinfo.io/json")
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
logger.warning(f"Proxy test failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Proxy test failed: {e}")
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'ip': data.get('ip'),
|
||||
'country': data.get('country'),
|
||||
'city': data.get('city'),
|
||||
'org': data.get('org'),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/config/location")
|
||||
async def set_location_config(
|
||||
config: LocationConfigRequest,
|
||||
|
|
|
|||
|
|
@ -44,27 +44,38 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
|
|||
|
||||
backend_type = result.config_value if result and result.config_value else 'playwright_local'
|
||||
|
||||
if backend_type == 'playwright_local':
|
||||
return PlaywrightLocalBackend()
|
||||
|
||||
elif backend_type == 'playwright_proxy':
|
||||
# Get proxy config
|
||||
proxy_result = db.execute(
|
||||
text("""
|
||||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password')
|
||||
""")
|
||||
)
|
||||
proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()}
|
||||
return PlaywrightLocalBackend(proxy_config=proxy_config)
|
||||
|
||||
elif backend_type == 'apify':
|
||||
if backend_type == 'apify':
|
||||
# Future: Apify backend
|
||||
raise NotImplementedError("Apify backend not yet implemented")
|
||||
|
||||
else:
|
||||
if backend_type not in ('playwright_local', 'playwright_proxy'):
|
||||
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
|
||||
return PlaywrightLocalBackend()
|
||||
|
||||
# Proxy configuration is managed in system_config (Settings page). When the
|
||||
# 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
|
||||
# the backend falls back to the BOOKING_PROXY_* environment variables.
|
||||
proxy_rows = db.execute(
|
||||
text("""
|
||||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key LIKE 'booking_proxy_%'
|
||||
""")
|
||||
).fetchall()
|
||||
proxy = {row.config_key: row.config_value for row in proxy_rows}
|
||||
|
||||
if 'booking_proxy_enabled' in proxy:
|
||||
if proxy.get('booking_proxy_enabled') == 'true':
|
||||
return PlaywrightLocalBackend(proxy_config={
|
||||
'host': proxy.get('booking_proxy_host', ''),
|
||||
'port': proxy.get('booking_proxy_port', '823'),
|
||||
'username': proxy.get('booking_proxy_username', ''),
|
||||
'password': proxy.get('booking_proxy_password', ''),
|
||||
'country': proxy.get('booking_proxy_country', 'gb'),
|
||||
})
|
||||
# Explicitly disabled in the DB — direct connection, ignore env.
|
||||
return PlaywrightLocalBackend(proxy_config={})
|
||||
|
||||
# No DB override — let the backend read BOOKING_PROXY_* env vars.
|
||||
return PlaywrightLocalBackend()
|
||||
|
||||
|
||||
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue