Manage scraper proxy from the Settings page

- New "Scraper Proxy" tab: enable toggle, host/port/username/password/
  country, Save, and a Test Connection button that reports the live exit
  IP + country through the proxy
- Backend proxy config now lives in system_config (DB authoritative when
  booking_proxy_enabled is set; BOOKING_PROXY_* env vars are the fallback)
- Dedicated /config/proxy GET/POST/test endpoints; password is write-only
  (never returned, blank keeps the stored value) and masked in /config/system
- Surface proxy status keys in the read-only System tab

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 20:36:19 +00:00
parent 11b995739c
commit 270d8293d1
3 changed files with 321 additions and 21 deletions

View file

@ -44,27 +44,38 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
backend_type = result.config_value if result and result.config_value else 'playwright_local'
if backend_type == 'playwright_local':
return PlaywrightLocalBackend()
elif backend_type == 'playwright_proxy':
# Get proxy config
proxy_result = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password')
""")
)
proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()}
return PlaywrightLocalBackend(proxy_config=proxy_config)
elif backend_type == 'apify':
if backend_type == 'apify':
# Future: Apify backend
raise NotImplementedError("Apify backend not yet implemented")
else:
if backend_type not in ('playwright_local', 'playwright_proxy'):
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
return PlaywrightLocalBackend()
# Proxy configuration is managed in system_config (Settings page). When the
# 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
# the backend falls back to the BOOKING_PROXY_* environment variables.
proxy_rows = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key LIKE 'booking_proxy_%'
""")
).fetchall()
proxy = {row.config_key: row.config_value for row in proxy_rows}
if 'booking_proxy_enabled' in proxy:
if proxy.get('booking_proxy_enabled') == 'true':
return PlaywrightLocalBackend(proxy_config={
'host': proxy.get('booking_proxy_host', ''),
'port': proxy.get('booking_proxy_port', '823'),
'username': proxy.get('booking_proxy_username', ''),
'password': proxy.get('booking_proxy_password', ''),
'country': proxy.get('booking_proxy_country', 'gb'),
})
# Explicitly disabled in the DB — direct connection, ignore env.
return PlaywrightLocalBackend(proxy_config={})
# No DB override — let the backend read BOOKING_PROXY_* env vars.
return PlaywrightLocalBackend()
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: