Add hotel-page scraper backend with full room/rate plan extraction

Switches booking.com scraping from search-results page (Cloudflare-targeted)
to individual hotel property pages (not CF-protected). Each page load returns
all room types, all rate plan variants (room-only/B&B × refundable/non-ref ×
1-2 adults), and availability counts.

Key changes:
- New PlaywrightHotelPageBackend: proxy reuse until block, rotate on CF/WAF
- booking_scraper.py: _run_hotel_page_scrape(), scrape_hotel_date(),
  _scrape_hotels_concurrent() — sharded by hotel so one proxy session covers
  all dates for one hotel (looks human)
- schema.sql: ADD COLUMN rate_plan_id, max_persons on booking_com_rates
- competitors API: filter max_persons=2, order by rate_gross ASC as tiebreaker
  so DISTINCT ON returns cheapest 2-adult rate from latest batch

Enable via Settings → Scraper Backend → playwright_hotel_page

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 23:08:03 +00:00
parent b10b2f5990
commit bb37fcf501
6 changed files with 608 additions and 8 deletions

View file

@ -25,7 +25,10 @@ from sqlalchemy.orm import Session
from database import SyncSessionLocal
from services import proxy as proxy_util
from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
from .scraper_backends import (
ScraperBackend, PlaywrightLocalBackend, PlaywrightHotelPageBackend,
HotelData, RateData, AvailabilityStatus,
)
logger = logging.getLogger(__name__)
@ -94,12 +97,17 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
# Future: Apify backend
raise NotImplementedError("Apify backend not yet implemented")
proxy_cfg = proxy_util.load_config(db)
if backend_type == 'playwright_hotel_page':
return PlaywrightHotelPageBackend(proxy_config=proxy_cfg)
if backend_type not in ('playwright_local', 'playwright_proxy'):
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
# Proxy config resolved by the shared module: system_config is authoritative
# when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct).
return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db))
return PlaywrightLocalBackend(proxy_config=proxy_cfg)
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
@ -189,9 +197,11 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
text("""
INSERT INTO booking_com_rates
(hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id)
breakfast_included, free_cancellation, no_prepayment, rooms_left,
rate_plan_id, max_persons, scrape_batch_id)
VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
:breakfast, :cancel, :prepay, :rooms_left, :batch_id)
:breakfast, :cancel, :prepay, :rooms_left,
:rate_plan_id, :max_persons, :batch_id)
"""),
{
'hotel_id': hotel_id,
@ -204,6 +214,8 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
'cancel': rate.free_cancellation,
'prepay': rate.no_prepayment,
'rooms_left': rate.rooms_left,
'rate_plan_id': rate.rate_plan_id,
'max_persons': rate.max_persons,
'batch_id': str(batch_id),
}
)
@ -282,6 +294,69 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
return len(cleaned)
def get_active_hotels(db: Session) -> List[Dict[str, Any]]:
"""Return all active hotels that have a booking_com_url (needed for hotel-page scraping)."""
rows = db.execute(
text("""
SELECT id, booking_com_id, name, booking_com_url
FROM booking_com_hotels
WHERE is_active = TRUE AND booking_com_url IS NOT NULL
ORDER BY display_order, id
""")
).fetchall()
return [dict(r._mapping) for r in rows]
async def scrape_hotel_date(
db: Session,
hotel: Dict[str, Any],
rate_date: date,
backend: 'PlaywrightHotelPageBackend',
batch_id: uuid.UUID,
adults: int = 2,
max_retries: int = 2,
) -> Dict[str, Any]:
"""
Scrape all rate plans for one hotel on one date via the hotel page backend.
On block detection, the backend rotates its proxy context automatically.
We retry up to max_retries times to handle the rotation.
"""
check_out = rate_date + timedelta(days=1)
hotel_url = hotel['booking_com_url']
for attempt in range(max_retries + 1):
result = await backend.scrape_hotel_page(hotel_url, rate_date, check_out, adults)
if result.success:
rates_saved = 0
for rate in result.rates:
rate.rate_date = rate_date # ensure date is set
try:
save_rate(db, rate, hotel['id'], batch_id)
rates_saved += 1
except Exception as e:
logger.warning(f"Error saving rate plan for {hotel['name']} {rate_date}: {e}")
db.rollback()
db.commit()
return {'success': True, 'blocked': False, 'rates_count': rates_saved}
if result.blocked and attempt < max_retries:
logger.info(
f"Hotel {hotel['name']} {rate_date} blocked (attempt {attempt + 1}), retrying…"
)
continue
return {
'success': False,
'blocked': result.blocked,
'error': result.error_message,
'rates_count': 0,
}
return {'success': False, 'blocked': True, 'rates_count': 0}
async def scrape_date(
db: Session,
rate_date: date,
@ -438,6 +513,13 @@ def get_scraper_concurrency(db: Session) -> int:
return 6
def _is_hotel_page_mode(db: Session) -> bool:
row = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'")
).fetchone()
return (row and row.config_value) == 'playwright_hotel_page'
def _effective_concurrency(db: Session, n_jobs: int) -> int:
"""Clamp configured concurrency to the workload, and force serial when the
proxy is off N workers would share one IP and hammer it, worse than 1."""
@ -451,6 +533,82 @@ def _effective_concurrency(db: Session, n_jobs: int) -> int:
return max(1, min(configured, n_jobs))
async def _scrape_hotels_concurrent(
hotel_date_jobs: List[Tuple[Dict[str, Any], date]],
concurrency: int,
batch_id: uuid.UUID,
adults: int = 2,
) -> Dict[str, int]:
"""
Scrape (hotel, date) pairs using the hotel-page backend.
Jobs are sharded by HOTEL (not date) so each worker keeps its proxy session
alive across all dates for one hotel looks like a single user checking
availability for a trip.
"""
# Group by hotel to get stable shards
hotels_seen: List[Dict[str, Any]] = []
hotel_dates: Dict[int, List[date]] = {}
for hotel, rate_date in hotel_date_jobs:
hid = hotel['id']
if hid not in hotel_dates:
hotel_dates[hid] = []
hotels_seen.append(hotel)
hotel_dates[hid].append(rate_date)
# Shard hotels across workers
shards: List[List[Dict[str, Any]]] = [hotels_seen[i::concurrency] for i in range(concurrency)]
shards = [s for s in shards if s]
async def worker(hotel_shard: List[Dict[str, Any]], widx: int) -> Dict[str, int]:
acc = {'rates': 0, 'completed': 0, 'failed': 0, 'blocked': 0}
wdb = SyncSessionLocal()
backend = PlaywrightHotelPageBackend(proxy_config=proxy_util.load_config(wdb))
try:
for hotel in hotel_shard:
for rate_date in hotel_dates[hotel['id']]:
try:
result = await scrape_hotel_date(wdb, hotel, rate_date, backend, batch_id, adults)
except Exception as e:
logger.error(f"[worker {widx}] {hotel['name']} {rate_date} crashed: {e}")
try:
wdb.rollback()
except Exception:
pass
acc['failed'] += 1
continue
if result.get('blocked'):
acc['blocked'] += 1
acc['failed'] += 1
elif result['success']:
acc['rates'] += result['rates_count']
acc['completed'] += 1
else:
acc['failed'] += 1
finally:
try:
await backend.close()
except Exception:
pass
wdb.close()
return acc
results = await asyncio.gather(
*(worker(shard, i) for i, shard in enumerate(shards)),
return_exceptions=True,
)
agg = {'rates': 0, 'completed': 0, 'failed': 0, 'blocked': 0}
for r in results:
if isinstance(r, Exception):
logger.error(f"Hotel-page scrape worker crashed: {r}")
continue
for k in agg:
agg[k] += r[k]
return agg
def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
"""mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
if queue_id is None:
@ -565,12 +723,105 @@ async def run_manual_scrape(
_release_scrape_lock()
async def _run_hotel_page_scrape(
db: Session,
from_date: date,
to_date: date,
scrape_type: str,
) -> Dict[str, Any]:
"""
Hotel-page scrape mode: load each known hotel's property page per date.
Unlike search-results mode, this scrapes known hotels (from booking_com_hotels
WHERE is_active AND booking_com_url IS NOT NULL) rather than discovering them
from search results. Workers are sharded by hotel so each worker's proxy session
covers all dates for one hotel before moving to the next.
"""
hotels = get_active_hotels(db)
if not hotels:
return {
'success': False,
'error': 'No active hotels with booking_com_url in database. '
'Run a search-results scrape first to populate hotels.',
}
dates: List[date] = []
current_date = from_date
while current_date <= to_date:
dates.append(current_date)
current_date += timedelta(days=1)
# Build (hotel, date) job pairs
hotel_date_jobs: List[Tuple[Dict[str, Any], date]] = [
(hotel, d) for hotel in hotels for d in dates
]
batch_id = create_scrape_batch(db, scrape_type)
db.execute(
text("UPDATE booking_scrape_log SET dates_queued = :n WHERE batch_id = :bid"),
{'n': len(hotel_date_jobs), 'bid': str(batch_id)}
)
db.commit()
concurrency = _effective_concurrency(db, len(hotels))
logger.info(
f"Hotel-page scrape {from_date}..{to_date}: "
f"{len(hotels)} hotels × {len(dates)} dates = {len(hotel_date_jobs)} jobs, "
f"{concurrency} worker(s)"
)
try:
config = get_scrape_config(db)
adults = config['adults'] if config else 2
agg = await _scrape_hotels_concurrent(hotel_date_jobs, concurrency, batch_id, adults)
status = 'completed' if agg['completed'] else 'failed'
update_scrape_batch(
db, batch_id,
status=status,
hotels_found=len(hotels),
rates_scraped=agg['rates'],
)
db.execute(
text("""
UPDATE booking_scrape_log SET
dates_completed = :completed, dates_failed = :failed
WHERE batch_id = :bid
"""),
{'completed': agg['completed'], 'failed': agg['failed'], 'bid': str(batch_id)}
)
db.commit()
return {
'success': agg['completed'] > 0,
'blocked': agg['blocked'] > 0,
'dates_completed': agg['completed'],
'dates_failed': agg['failed'],
'hotels_found': len(hotels),
'rates_scraped': agg['rates'],
}
except Exception as e:
logger.error(f"Hotel-page scrape error: {e}")
update_scrape_batch(db, batch_id, status='failed', error_message=str(e))
return {
'success': False,
'error': str(e),
'dates_completed': 0,
'dates_failed': len(hotel_date_jobs),
}
async def _run_manual_scrape_locked(
db: Session,
from_date: date,
to_date: date
) -> Dict[str, Any]:
# Get config
# Hotel-page mode: iterate known hotels × dates
if _is_hotel_page_mode(db):
return await _run_hotel_page_scrape(db, from_date, to_date, 'manual')
# Search-results mode (original)
config = get_scrape_config(db)
if not config:
return {
@ -751,6 +1002,17 @@ async def process_queue(db: Session) -> Dict[str, Any]:
async def _process_queue_locked(db: Session) -> Dict[str, Any]:
# Hotel-page mode ignores the queue and just scrapes the queued date range directly
if _is_hotel_page_mode(db):
items = get_pending_queue_items(db, limit=200)
if not items:
return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'}
dates = [it['rate_date'] for it in items]
# Mark all as completed (hotel-page scrape manages its own tracking)
for it in items:
_safe_mark_queue(db, it['id'], 'completed')
return await _run_hotel_page_scrape(db, min(dates), max(dates), 'scheduled')
# Get config
config = get_scrape_config(db)
if not config:

View file

@ -9,6 +9,7 @@ Provides pluggable backends to allow switching between:
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
from .playwright_local import PlaywrightLocalBackend
from .playwright_hotel_page import PlaywrightHotelPageBackend
__all__ = [
'ScraperBackend',
@ -17,4 +18,5 @@ __all__ = [
'RateData',
'AvailabilityStatus',
'PlaywrightLocalBackend',
'PlaywrightHotelPageBackend',
]

View file

@ -36,6 +36,8 @@ class RateData:
no_prepayment: Optional[bool] = None
rooms_left: Optional[int] = None # "Only X rooms left"
available_qty: Optional[int] = None # Future: from hotel page dropdown
rate_plan_id: Optional[str] = None # block_id from hotel page (identifies rate variant)
max_persons: Optional[int] = None # Occupancy this rate applies to
@dataclass

View file

@ -0,0 +1,328 @@
"""
Booking.com Hotel Page Scraper Backend
Scrapes individual hotel property pages instead of search results.
Returns all room types and rate plan variants per hotel per date.
Advantages over search-results approach:
- Individual hotel pages are not Cloudflare-protected (search results are)
- Captures every room type, rate plan, meal plan, and availability count
- Proxy reuse: one residential IP stays valid across many hotel pages
(looks like a human browsing properties), rotates only when blocked
Rate plan variants per room type (typical):
2-adult + room only + non-refundable
2-adult + room only + free cancellation
2-adult + breakfast + non-refundable
2-adult + breakfast + free cancellation
1-adult + room only / breakfast (filtered out by caller if not needed)
"""
import asyncio
import logging
import re
from datetime import date
from decimal import Decimal
from typing import List, Optional, Tuple
from urllib.parse import urlparse
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
logger = logging.getLogger(__name__)
# JS that extracts all rate plan rows from the room availability table.
# Runs inside the page after the room table has loaded.
_EXTRACT_RATES_JS = """
() => {
const results = [];
document.querySelectorAll('[id^="room_type_id_"]').forEach(roomEl => {
const roomId = roomEl.getAttribute('data-room-id') || roomEl.id.replace('room_type_id_', '');
const roomName = (
roomEl.querySelector('.hprt-roomtype-icon-link')?.innerText ||
roomEl.querySelector('span')?.innerText || ''
).trim();
// Availability count from scarcity indicator ("We have 2 left")
const availText = roomEl.closest('tr')
?.querySelector('.only_x_left, .thisRoomAvailabilityNew span')
?.innerText?.trim() || '';
const availMatch = availText.match(/\\d+/);
const availCount = availMatch ? parseInt(availMatch[0]) : null;
// Walk sibling <tr> rows that belong to this room type
let tr = roomEl.closest('tr');
while (tr) {
if (tr.classList.contains('js-rt-block-row')) {
const blockId = tr.getAttribute('data-block-id') || '';
const priceRaw = tr.getAttribute('data-hotel-rounded-price') || '';
let fltrs = {};
try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {}
// Conditions cell (3rd <td>) holds meal plan + cancel info
const cells = tr.querySelectorAll('td');
const condCell = cells.length >= 3 ? cells[2].innerText || '' : '';
const breakfastIncluded = condCell.toLowerCase().includes('breakfast');
const nonRefundable = (fltrs.non_refundable === 1);
// Free cancellation date: "Free cancellation before DD Month YYYY"
const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
// Persons: first <td> ("Max persons: 2" or "Only for 1 guest")
const personsCell = cells.length >= 1 ? cells[0].innerText || '' : '';
const personsMatch = personsCell.match(/\\d+/);
const maxPersons = personsMatch ? parseInt(personsMatch[0]) : null;
results.push({
room_id: roomId,
room_name: roomName,
avail_count: availCount,
block_id: blockId,
price: priceRaw ? parseInt(priceRaw) : null,
breakfast_included: breakfastIncluded,
non_refundable: nonRefundable,
free_cancel_text: freeCancelText,
max_persons: maxPersons,
});
}
tr = tr.nextElementSibling;
if (!tr) break;
// Stop at the next room type's header row
if (tr.querySelector('[id^="room_type_id_"]')) break;
}
});
return results;
}
"""
def _slug_from_url(url: str) -> Optional[str]:
"""Extract hotel slug from Booking.com hotel page URL."""
if not url:
return None
m = re.search(r'/hotel/\w+/([^.]+)\.', url)
return m.group(1) if m else None
def _build_hotel_url(hotel_url: str, checkin: date, checkout: date, adults: int) -> str:
base = hotel_url.split('?')[0]
return (
f"{base}?checkin={checkin}&checkout={checkout}"
f"&group_adults={adults}&no_rooms=1&selected_currency=GBP"
)
class PlaywrightHotelPageBackend(ScraperBackend):
"""
Scrapes individual Booking.com hotel property pages.
Keeps one browser context alive across multiple hotel+date requests.
Rotates proxy only when a CF/WAF block is detected.
"""
def __init__(self, proxy_config: dict = None):
self._proxy_config = proxy_config or {}
self._pw = None
self._browser: Optional[Browser] = None
self._context: Optional[BrowserContext] = None
self._requests_on_context = 0
# Rotate after this many requests even without a block (keeps session fresh)
self._max_requests_per_context = 40
def _proxy_enabled(self) -> bool:
return bool(self._proxy_config.get('server'))
def _proxy_kwargs(self) -> dict:
if not self._proxy_enabled():
return {}
cfg = self._proxy_config
proxy = {'server': cfg['server']}
if cfg.get('username'):
proxy['username'] = cfg['username']
if cfg.get('password'):
proxy['password'] = cfg['password']
return {'proxy': proxy}
async def _start(self):
if not self._pw:
self._pw = await async_playwright().start()
if not self._browser:
self._browser = await self._pw.chromium.launch(
headless=True,
args=['--no-sandbox', '--disable-dev-shm-usage'],
)
async def _get_context(self) -> BrowserContext:
await self._start()
if self._context is None or self._requests_on_context >= self._max_requests_per_context:
if self._context:
try:
await self._context.close()
except Exception:
pass
self._context = await self._browser.new_context(
user_agent=(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/125.0.0.0 Safari/537.36'
),
**self._proxy_kwargs(),
)
self._requests_on_context = 0
logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else ""))
return self._context
async def _rotate_context(self):
"""Force a new browser context (new proxy session)."""
if self._context:
try:
await self._context.close()
except Exception:
pass
self._context = None
self._requests_on_context = 0
logger.info("Rotated browser context (new proxy session)")
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2,
) -> ScraperResult:
"""
Load a hotel page and extract all room types + rate plan variants.
On block detection, rotates proxy and returns blocked=True so the
caller can retry with the fresh context.
"""
page_url = _build_hotel_url(hotel_url, check_in, check_out, adults)
context = await self._get_context()
self._requests_on_context += 1
page: Optional[Page] = None
try:
page = await context.new_page()
# Load the page
page_loaded = False
try:
await page.goto(page_url, wait_until='domcontentloaded', timeout=30000)
page_loaded = True
except Exception as e:
logger.warning(f"goto timeout for {hotel_url} {check_in}: {e}")
if not page_loaded:
return ScraperResult(success=False, blocked=False, error_message='page load timeout')
# Wait for the room table (prices render via JS after DOM)
room_table_appeared = False
try:
await page.wait_for_selector('[id^="room_type_id_"]', timeout=15000)
room_table_appeared = True
except Exception:
pass
if not room_table_appeared:
# Could be sold-out, no-availability, or a block page
content = await page.content()
is_blocked, reason = self.detect_blocking(content)
if is_blocked or len(content) < 20000:
logger.warning(f"Block detected for {hotel_url} {check_in}: {reason or 'small page'}")
await self._rotate_context()
return ScraperResult(success=False, blocked=True, block_reason=reason or 'small page')
# No availability for this date
return ScraperResult(
success=True,
rates=[RateData(
rate_date=check_in,
availability_status=AvailabilityStatus.SOLD_OUT,
currency='GBP',
)],
)
# Extra wait for JS prices to render
await asyncio.sleep(10)
# Check for explicit "no availability" message
no_avail_el = await page.query_selector('#no_availability_msg')
if no_avail_el and await no_avail_el.is_visible():
return ScraperResult(
success=True,
rates=[RateData(
rate_date=check_in,
availability_status=AvailabilityStatus.SOLD_OUT,
currency='GBP',
)],
)
# Extract all rate plan rows
raw_plans = await page.evaluate(_EXTRACT_RATES_JS)
if not raw_plans:
return ScraperResult(success=True, rates=[])
rates: List[RateData] = []
for plan in raw_plans:
if plan['price'] is None:
continue
rates.append(RateData(
rate_date=check_in,
availability_status=AvailabilityStatus.AVAILABLE,
rate_gross=Decimal(plan['price']),
currency='GBP',
room_type=plan['room_name'] or None,
breakfast_included=plan['breakfast_included'],
free_cancellation=not plan['non_refundable'],
rooms_left=plan['avail_count'],
rate_plan_id=plan['block_id'] or None,
max_persons=plan['max_persons'],
))
logger.info(
f"Hotel page {_slug_from_url(hotel_url)} {check_in}: "
f"{len(raw_plans)} rate plans extracted"
)
return ScraperResult(success=True, rates=rates)
except Exception as e:
logger.error(f"scrape_hotel_page error for {hotel_url} {check_in}: {e}")
return ScraperResult(success=False, error_message=str(e))
finally:
if page:
try:
await page.close()
except Exception:
pass
async def scrape_location_search(self, *args, **kwargs) -> ScraperResult:
"""Not used by this backend — hotel pages replace location search."""
raise NotImplementedError("PlaywrightHotelPageBackend does not support location search")
async def close(self):
if self._context:
try:
await self._context.close()
except Exception:
pass
self._context = None
if self._browser:
try:
await self._browser.close()
except Exception:
pass
self._browser = None
if self._pw:
try:
await self._pw.stop()
except Exception:
pass
self._pw = None