hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Extract shared proxy module; make it available to direct scraper; remove dead code

Browse source 
Proxy config, DataImpulse sticky-session username syntax, and Playwright/
httpx proxy builders now live in one place (services/proxy.py) instead of
being duplicated across the Booking.com backend and the /config/proxy test
endpoint. Both scrapers consume it.

- services/proxy.py: load_config/normalize (DB-authoritative, env fallback),
  new_session_id, username, playwright_proxy, httpx_proxy_url
- PlaywrightLocalBackend delegates proxy building to the module
- get_scraper_backend factory uses proxy.load_config (one resolution path)
- test_proxy_config endpoint uses the shared URL builder; httpx proxies= ->
  proxy= (forward-compatible, 0.28-safe)
- Direct booking-engine scraper (httpx) can now route through the same proxy,
  gated by the direct_scraper_use_proxy flag (default off, plumbing ready)

Dead code removed: set_scraper_paused (never called — rotate-on-block
replaced pause-on-block), get_competitor_matrix / get_hotels_list /
update_hotel_tier (endpoints have their own SQL), unused PROXY_KEYS tuple.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 22:55:24 +00:00
parent 6b7f00b40a
commit 1dc8c0a945
5 changed files with 147 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

146
backend/services/booking_scraper.py
View file

@ -23,6 +23,7 @@ from sqlalchemy import text
from sqlalchemy.orm import Session
from database import SyncSessionLocal
from services import proxy as proxy_util
from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
logger = logging.getLogger(__name__)
@ -53,31 +54,9 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
if backend_type not in ('playwright_local', 'playwright_proxy'):
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
# Proxy configuration is managed in system_config (Settings page). When the
# 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
# the backend falls back to the BOOKING_PROXY_* environment variables.
proxy_rows = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key LIKE 'booking_proxy_%'
""")
).fetchall()
proxy = {row.config_key: row.config_value for row in proxy_rows}
if 'booking_proxy_enabled' in proxy:
if proxy.get('booking_proxy_enabled') == 'true':
return PlaywrightLocalBackend(proxy_config={
'host': proxy.get('booking_proxy_host', ''),
'port': proxy.get('booking_proxy_port', '823'),
'username': proxy.get('booking_proxy_username', ''),
'password': proxy.get('booking_proxy_password', ''),
'country': proxy.get('booking_proxy_country', 'gb'),
})
# Explicitly disabled in the DB — direct connection, ignore env.
return PlaywrightLocalBackend(proxy_config={})
# No DB override — let the backend read BOOKING_PROXY_* env vars.
return PlaywrightLocalBackend()
# Proxy config resolved by the shared module: system_config is authoritative
# when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct).
return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db))
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
@ -135,21 +114,6 @@ async def is_scraper_paused(db: Session) -> bool:
return True
async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
"""Set scraper pause status."""
db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
{'val': 'true' if paused else 'false'}
)
if paused:
pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
{'val': pause_until}
)
db.commit()
def save_hotel(db: Session, hotel: HotelData) -> int:
"""
Save or update a hotel in the database.
@ -861,105 +825,3 @@ async def _process_queue_locked(db: Session) -> Dict[str, Any]:
'dates_failed': len(jobs),
}
def get_competitor_matrix(
db: Session,
from_date: date,
to_date: date,
include_market: bool = False
) -> List[Dict[str, Any]]:
"""
Get rate comparison matrix for competitors.
Args:
db: Database session
from_date: Start date
to_date: End date
include_market: Include market tier hotels
Returns:
List of rate records for matrix display
"""
tier_filter = "h.tier IN ('own', 'competitor')"
if include_market:
tier_filter = "h.tier IN ('own', 'competitor', 'market')"
result = db.execute(
text(f"""
SELECT
r.rate_date,
h.id AS hotel_id,
h.name AS hotel_name,
h.tier,
h.display_order,
h.star_rating,
h.review_score,
r.availability_status,
r.rate_gross,
r.room_type,
r.breakfast_included,
r.free_cancellation,
r.no_prepayment,
r.rooms_left,
r.scraped_at
FROM booking_latest_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE {tier_filter}
AND h.is_active = TRUE
AND r.rate_date BETWEEN :from_date AND :to_date
ORDER BY r.rate_date, h.display_order, h.name
"""),
{'from_date': from_date, 'to_date': to_date}
)
return [dict(row._mapping) for row in result.fetchall()]
def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
"""
Get list of discovered hotels.
Args:
db: Database session
tier: Filter by tier ('own', 'competitor', 'market') or None for all
Returns:
List of hotel records
"""
where_clause = "WHERE is_active = TRUE"
if tier:
where_clause += f" AND tier = '{tier}'"
result = db.execute(
text(f"""
SELECT
id, booking_com_id, name, booking_com_url,
star_rating, review_score, review_count,
tier, display_order, notes,
first_seen_at, last_seen_at
FROM booking_com_hotels
{where_clause}
ORDER BY display_order, name
""")
)
return [dict(row._mapping) for row in result.fetchall()]
def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
"""Update a hotel's tier and display order."""
if tier not in ('own', 'competitor', 'market'):
raise ValueError(f"Invalid tier: {tier}")
params = {'hotel_id': hotel_id, 'tier': tier}
set_clause = "tier = :tier"
if display_order is not None:
set_clause += ", display_order = :order"
params['order'] = display_order
db.execute(
text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
params
)
db.commit()

16
backend/services/direct_scraper.py
View file

@ -11,6 +11,7 @@ import httpx
from sqlalchemy import text
from database import SyncSessionLocal
from services import proxy as proxy_util
from services.direct_profiles import get_profile
log = logging.getLogger(__name__)
@ -18,6 +19,17 @@ log = logging.getLogger(__name__)
REQUEST_DELAY = 10.0
DISCOVERY_DELAY = 2.0
def _direct_proxy_url() -> str | None:
"""Proxy URL for direct-engine scraping, or None. Shares the Booking.com
proxy config but is gated by the `direct_scraper_use_proxy` flag (default
off — booking-engine APIs usually don't need it). Ready to switch on."""
db = SyncSessionLocal()
try:
return proxy_util.direct_httpx_proxy(db)
finally:
db.close()
# Track running discovery scrapes: hotel_id -> status dict
_discovery_status: dict[int, dict] = {}
@ -50,7 +62,7 @@ async def run_discovery(hotel_id: int, profile_name: str, params: dict):
found_rooms: set[str] = set()
found_rates: set[str] = set()
async with httpx.AsyncClient() as client:
async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client:
for i, arrival in enumerate(dates):
await asyncio.sleep(DISCOVERY_DELAY)
try:
@ -127,7 +139,7 @@ def run_scrape(hotel_id: int, profile_name: str, params: dict):
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
async with httpx.AsyncClient() as client:
async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client:
try:
arrival_dates = await profile.fetch_arrival_dates(client, params)
except Exception as e:

108
backend/services/proxy.py Normal file
View file

@ -0,0 +1,108 @@
"""
Shared residential-proxy configuration for every scraper.
Single source of truth for:
- where proxy config lives (system_config `booking_proxy_*` keys, with
BOOKING_PROXY_* env fallback),
- the DataImpulse sticky-session username syntax (LOGIN__cr.<cc>;sessid.<id>),
- building proxy settings for both Playwright (launch dict) and httpx (URL).
The Booking.com browser scraper (Playwright) and the direct booking-engine
scraper (httpx) both consume this, so proxy logic is never duplicated. Direct
scraping is opt-in via the `direct_scraper_use_proxy` config key (default off —
booking-engine APIs generally don't need it), but the plumbing is ready.
"""
import os
import random
from typing import Optional
def new_session_id() -> str:
"""A fresh sticky-session id. DataImpulse pins one IP per id, so a new id
yields a new residential IP."""
return f"hnf{random.randint(100000, 999999)}"
def config_from_env() -> dict:
"""Proxy config from BOOKING_PROXY_* env vars ({} when unset)."""
host = os.getenv("BOOKING_PROXY_HOST", "").strip()
if not host:
return {}
return {
'host': host,
'port': os.getenv("BOOKING_PROXY_PORT", "823").strip(),
'username': os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
'password': os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
'country': os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
}
def normalize(raw: dict) -> dict:
"""Turn raw `booking_proxy_*` system_config values into a proxy config dict.
DB is authoritative when `booking_proxy_enabled` is present; otherwise fall
back to env. Returns {} when disabled or unconfigured."""
if 'booking_proxy_enabled' in raw:
if raw.get('booking_proxy_enabled') != 'true':
return {}
return {
'host': (raw.get('booking_proxy_host') or '').strip(),
'port': (raw.get('booking_proxy_port') or '823').strip(),
'username': (raw.get('booking_proxy_username') or '').strip(),
'password': (raw.get('booking_proxy_password') or '').strip(),
'country': (raw.get('booking_proxy_country') or 'gb').strip(),
}
return config_from_env()
def load_config(db) -> dict:
"""Resolve proxy config using a sync SQLAlchemy session. Returns {} when
disabled/unconfigured. (Async callers should fetch the rows themselves and
pass them through normalize().)"""
from sqlalchemy import text
rows = db.execute(
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'booking_proxy_%'")
).fetchall()
return normalize({r.config_key: r.config_value for r in rows})
def is_enabled(cfg: dict) -> bool:
return bool(cfg.get('host') and cfg.get('username'))
def username(cfg: dict, session_id: Optional[str] = None) -> str:
"""DataImpulse username: LOGIN__cr.<country>[;sessid.<id>]."""
user = f"{cfg['username']}__cr.{cfg.get('country', 'gb')}"
if session_id:
user += f";sessid.{session_id}"
return user
def playwright_proxy(cfg: dict, session_id: Optional[str] = None) -> Optional[dict]:
"""Proxy dict for chromium.launch(proxy=...). None when disabled."""
if not is_enabled(cfg):
return None
return {
'server': f"http://{cfg['host']}:{cfg['port']}",
'username': username(cfg, session_id),
'password': cfg['password'],
}
def httpx_proxy_url(cfg: dict, session_id: Optional[str] = None) -> Optional[str]:
"""Proxy URL for httpx.AsyncClient(proxies=...). None when disabled."""
if not is_enabled(cfg):
return None
return f"http://{username(cfg, session_id)}:{cfg['password']}@{cfg['host']}:{cfg['port']}"
def direct_httpx_proxy(db) -> Optional[str]:
"""Proxy URL for the direct booking-engine scraper — only when the proxy is
configured AND `direct_scraper_use_proxy` is explicitly enabled (default off).
Uses a fresh session id per call so direct runs spread across IPs."""
from sqlalchemy import text
flag = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'direct_scraper_use_proxy'")
).fetchone()
if not flag or flag.config_value != 'true':
return None
return httpx_proxy_url(load_config(db), new_session_id())

40
backend/services/scraper_backends/playwright_local.py
View file

@ -7,7 +7,6 @@ No proxy - direct connection. Suitable for low-volume scraping.
import asyncio
import logging
import os
import random
import re
from datetime import date
@ -15,6 +14,8 @@ from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode, urlparse, parse_qs
from services import proxy as proxy_util
def location_params_from_url(url: str) -> dict:
"""Pull the stable destination params (ss, dest_id, dest_type) out of a
@ -109,8 +110,9 @@ class PlaywrightLocalBackend(ScraperBackend):
Args:
proxy_config: Optional proxy configuration. Falls back to the
BOOKING_PROXY_* environment variables when not supplied.
See services/proxy.py for the shared config/build helpers.
"""
self.proxy_config = proxy_config if proxy_config is not None else self._proxy_from_env()
self.proxy_config = proxy_config if proxy_config is not None else proxy_util.config_from_env()
self._playwright = None
self._browser: Optional[Browser] = None
self._context: Optional[BrowserContext] = None
@ -122,40 +124,12 @@ class PlaywrightLocalBackend(ScraperBackend):
f"(country={self.proxy_config.get('country', 'gb')}, session={self._session_id})"
)
@staticmethod
def _proxy_from_env() -> dict:
"""Read proxy settings from BOOKING_PROXY_* env vars (empty = disabled)."""
host = os.getenv("BOOKING_PROXY_HOST", "").strip()
if not host:
return {}
return {
"host": host,
"port": os.getenv("BOOKING_PROXY_PORT", "823").strip(),
"username": os.getenv("BOOKING_PROXY_USERNAME", "").strip(),
"password": os.getenv("BOOKING_PROXY_PASSWORD", "").strip(),
"country": os.getenv("BOOKING_PROXY_COUNTRY", "gb").strip(),
}
def _proxy_enabled(self) -> bool:
return bool(self.proxy_config.get("host") and self.proxy_config.get("username"))
return proxy_util.is_enabled(self.proxy_config)
def _new_session_id(self):
"""Pick a fresh sticky-session id — DataImpulse pins one IP per id."""
self._session_id = f"hnf{random.randint(100000, 999999)}"
def _proxy_launch_arg(self) -> Optional[dict]:
"""Build Playwright's proxy dict, encoding country + sticky session
into the username per DataImpulse's syntax: LOGIN__cr.gb;sessid.ID."""
if not self._proxy_enabled():
return None
username = f"{self.proxy_config['username']}__cr.{self.proxy_config.get('country', 'gb')}"
if self._session_id:
username += f";sessid.{self._session_id}"
return {
"server": f"http://{self.proxy_config['host']}:{self.proxy_config['port']}",
"username": username,
"password": self.proxy_config["password"],
}
self._session_id = proxy_util.new_session_id()
async def rotate_session(self):
"""Burn the current proxy IP and warm cache; the next scrape gets a
@ -198,7 +172,7 @@ class PlaywrightLocalBackend(ScraperBackend):
# Proxy is set at launch (Chromium binds the sticky session, which
# lives in the username, at the network layer). Rotating the IP
# therefore relaunches the browser — see rotate_session().
proxy = self._proxy_launch_arg()
proxy = proxy_util.playwright_proxy(self.proxy_config, self._session_id)
if proxy:
launch_kwargs['proxy'] = proxy
self._browser = await self._playwright.chromium.launch(**launch_kwargs)