hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add Rate Monitor app — Booking.com + direct booking engine competitor rates

Browse source 
Combines Booking.com Playwright scraper (from forecasting), direct booking
engine scraper (ported from laptop-archive/guestline-monitor), and Newbook
own-hotel rates into one focused tool. Four views: Bookability, Market View
(with price index badges + direct rate sub-rows), Direct Rates (per-competitor
room breakdown, min-stay flags, hotel config/discovery), Rate Analysis
(advance purchase curve, DOW chart, rate timeline, comparison table).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 12:06:30 +00:00
commit e05054172f
50 changed files with 11860 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

20
backend/services/scraper_backends/__init__.py Normal file
View file

@ -0,0 +1,20 @@
"""
Scraper backends for booking.com rate scraping.
Provides pluggable backends to allow switching between:
- playwright_local: Direct Playwright (default)
- playwright_proxy: Playwright with rotating proxies (future)
- apify_backend: Apify scraping service (future)
"""
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
from .playwright_local import PlaywrightLocalBackend
__all__ = [
'ScraperBackend',
'ScraperResult',
'HotelData',
'RateData',
'AvailabilityStatus',
'PlaywrightLocalBackend',
]

152
backend/services/scraper_backends/base.py Normal file
View file

@ -0,0 +1,152 @@
"""
Abstract base class for booking.com scraper backends.
Defines the interface that all scraper backends must implement,
allowing easy switching between local Playwright, proxied Playwright,
or external services like Apify.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import date
from decimal import Decimal
from typing import List, Optional, Dict, Any
from enum import Enum
class AvailabilityStatus(str, Enum):
"""Availability status for a hotel rate."""
AVAILABLE = 'available' # Rate found, bookable
SOLD_OUT = 'sold_out' # Hotel shows no availability
NO_DATA = 'no_data' # Couldn't determine (scraper issue)
@dataclass
class RateData:
"""Rate data for a single hotel on a single date."""
hotel_id: Optional[str] = None # Our internal hotel_id (filled after DB lookup)
booking_com_id: str = '' # Hotel ID from booking.com
rate_date: date = None
availability_status: AvailabilityStatus = AvailabilityStatus.NO_DATA
rate_gross: Optional[Decimal] = None
currency: str = 'GBP'
room_type: Optional[str] = None
breakfast_included: Optional[bool] = None
free_cancellation: Optional[bool] = None
no_prepayment: Optional[bool] = None
rooms_left: Optional[int] = None # "Only X rooms left"
available_qty: Optional[int] = None # Future: from hotel page dropdown
@dataclass
class HotelData:
"""Hotel data discovered from search results."""
booking_com_id: str
name: str
booking_com_url: Optional[str] = None
star_rating: Optional[Decimal] = None
review_score: Optional[Decimal] = None
review_count: Optional[int] = None
@dataclass
class ScraperResult:
"""Result from a scraping operation."""
success: bool
blocked: bool = False # True if anti-scrape blocking detected
block_reason: Optional[str] = None # CAPTCHA, rate limit, etc.
hotels: List[HotelData] = field(default_factory=list)
rates: List[RateData] = field(default_factory=list)
error_message: Optional[str] = None
page_content_sample: Optional[str] = None # For debugging
class ScraperBackend(ABC):
"""
Abstract base class for scraper backends.
All backends must implement these methods to provide a consistent
interface for the main booking_scraper.py service.
"""
# Common block detection signals
BLOCK_SIGNALS = [
'captcha',
'unusual traffic',
'access denied',
'please verify',
'too many requests',
'are you a robot',
'verify you are human',
'security check',
]
@abstractmethod
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name (e.g., "Bowness-on-Windermere")
check_in: Check-in date
check_out: Check-out date (typically check_in + 1 for single night)
adults: Number of adults for search
pages: Number of search result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
pass
@abstractmethod
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape an individual hotel page for detailed rates.
Future expansion - not used in initial implementation.
Will provide available_qty from room dropdowns.
Args:
hotel_url: Full booking.com URL for the hotel
check_in: Check-in date
check_out: Check-out date
adults: Number of adults
Returns:
ScraperResult with detailed rate information
"""
pass
@abstractmethod
async def close(self):
"""Clean up any resources (browser instances, etc.)."""
pass
def detect_blocking(self, page_content: str) -> tuple[bool, Optional[str]]:
"""
Check if page content shows anti-scrape response.
Args:
page_content: HTML content of the page
Returns:
Tuple of (is_blocked, reason)
"""
content_lower = page_content.lower()
for signal in self.BLOCK_SIGNALS:
if signal in content_lower:
return True, signal
return False, None

401
backend/services/scraper_backends/playwright_local.py Normal file
View file

@ -0,0 +1,401 @@
"""
Local Playwright backend for booking.com scraping.
Uses Playwright with Chromium to scrape search results.
No proxy - direct connection. Suitable for low-volume scraping.
"""
import asyncio
import logging
import random
import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import (
ScraperBackend,
ScraperResult,
HotelData,
RateData,
AvailabilityStatus
)
logger = logging.getLogger(__name__)
class PlaywrightLocalBackend(ScraperBackend):
"""
Local Playwright backend using Chromium.
Features:
- Rotates user agents
- Random delays between requests
- Mimics human scroll behavior
- Uses data-testid selectors for stability
"""
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
]
def __init__(self, proxy_config: dict = None):
"""
Initialize the backend.
Args:
proxy_config: Optional proxy configuration (for future use)
"""
self.proxy_config = proxy_config
self._playwright = None
self._browser: Optional[Browser] = None
async def _ensure_browser(self) -> Browser:
"""Ensure browser is running, start if needed."""
if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--no-sandbox',
'--disable-dev-shm-usage',
]
)
return self._browser
async def _create_context(self) -> BrowserContext:
"""Create a new browser context with random user agent."""
browser = await self._ensure_browser()
context = await browser.new_context(
user_agent=random.choice(self.USER_AGENTS),
viewport={'width': 1920, 'height': 1080},
locale='en-GB',
timezone_id='Europe/London',
)
return context
def _build_search_url(
self,
location: str,
check_in: date,
check_out: date,
adults: int,
offset: int = 0
) -> str:
"""Build booking.com search URL with parameters."""
params = {
'ss': location,
'checkin': check_in.isoformat(),
'checkout': check_out.isoformat(),
'group_adults': adults,
'no_rooms': 1,
'group_children': 0,
}
if offset > 0:
params['offset'] = offset
return f"https://www.booking.com/searchresults.en-gb.html?{urlencode(params)}"
def _parse_price(self, price_text: str) -> Optional[Decimal]:
"""Parse price from text like '£150' or 'GBP 150'."""
if not price_text:
return None
# Remove currency symbols and extract number
cleaned = re.sub(r'[£$€,\s]', '', price_text)
# Find first number (including decimals)
match = re.search(r'[\d,]+(?:\.\d{2})?', cleaned)
if match:
try:
return Decimal(match.group().replace(',', ''))
except InvalidOperation:
return None
return None
def _extract_hotel_id(self, url: str) -> Optional[str]:
"""Extract hotel ID from booking.com URL."""
if not url:
return None
# URL format: /hotel/gb/hotel-name.en-gb.html or ?dest_id=123
# Try to extract from URL path
match = re.search(r'/hotel/[a-z]{2}/([^/]+)\.', url)
if match:
return match.group(1)
# Try dest_id parameter
match = re.search(r'dest_id=(-?\d+)', url)
if match:
return match.group(1)
return None
async def _human_like_scroll(self, page: Page):
"""Simulate human-like scrolling behavior."""
# Scroll down in increments
for _ in range(3):
await page.mouse.wheel(0, random.randint(300, 600))
await asyncio.sleep(random.uniform(0.3, 0.8))
async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
"""Extract hotel and rate data from search results page."""
hotels = []
rates = []
# Wait for property cards - booking.com uses data-testid
try:
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
except Exception as e:
logger.warning(f"No property cards found: {e}")
return hotels, rates
# Get all property cards
cards = await page.query_selector_all('[data-testid="property-card"]')
logger.info(f"Found {len(cards)} property cards")
for card in cards:
try:
hotel = HotelData(booking_com_id='', name='')
rate = RateData(rate_date=rate_date)
# Hotel name
name_el = await card.query_selector('[data-testid="title"]')
if name_el:
hotel.name = (await name_el.inner_text()).strip()
if not hotel.name:
continue # Skip if no name found
# Hotel URL and ID
link_el = await card.query_selector('[data-testid="title-link"]')
if link_el:
hotel.booking_com_url = await link_el.get_attribute('href')
hotel.booking_com_id = self._extract_hotel_id(hotel.booking_com_url) or ''
rate.booking_com_id = hotel.booking_com_id
# Star rating - look for star icons or rating text
stars_el = await card.query_selector('[data-testid="rating-stars"]')
if stars_el:
stars_text = await stars_el.get_attribute('aria-label') or ''
match = re.search(r'(\d+)', stars_text)
if match:
hotel.star_rating = Decimal(match.group(1))
# Review score
score_el = await card.query_selector('[data-testid="review-score"]')
if score_el:
score_text = await score_el.inner_text()
match = re.search(r'([\d.]+)', score_text)
if match:
try:
hotel.review_score = Decimal(match.group(1))
except InvalidOperation:
pass
# Check for no availability message FIRST
no_avail_el = await card.query_selector('[data-testid="availability-message"]')
if no_avail_el:
avail_text = (await no_avail_el.inner_text()).lower()
if 'no availability' in avail_text or 'sold out' in avail_text:
rate.availability_status = AvailabilityStatus.SOLD_OUT
hotels.append(hotel)
rates.append(rate)
continue
# Price
price_el = await card.query_selector('[data-testid="price-and-discounted-price"]')
if not price_el:
# Try alternative selector
price_el = await card.query_selector('[data-testid="price"]')
if price_el:
price_text = await price_el.inner_text()
rate.rate_gross = self._parse_price(price_text)
if rate.rate_gross:
rate.availability_status = AvailabilityStatus.AVAILABLE
# Room type
room_el = await card.query_selector('[data-testid="recommended-units"]')
if room_el:
rate.room_type = (await room_el.inner_text()).strip()
# Rate option badges - try multiple selectors
# Breakfast included
breakfast_el = await card.query_selector('[data-testid="breakfast-included"]')
if not breakfast_el:
# Check text content for breakfast mentions
card_text = (await card.inner_text()).lower()
rate.breakfast_included = 'breakfast included' in card_text
else:
rate.breakfast_included = True
# Free cancellation
cancel_el = await card.query_selector('[data-testid="cancellation-policy"]')
if cancel_el:
cancel_text = (await cancel_el.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in cancel_text
else:
card_text = (await card.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in card_text
# No prepayment
prepay_el = await card.query_selector('[data-testid="no-prepayment"]')
if prepay_el:
rate.no_prepayment = True
else:
card_text = (await card.inner_text()).lower()
rate.no_prepayment = 'no prepayment' in card_text
# Rooms left / scarcity indicator
scarcity_el = await card.query_selector('[data-testid="availability-rate"]')
if scarcity_el:
scarcity_text = await scarcity_el.inner_text()
match = re.search(r'(\d+)\s*room', scarcity_text.lower())
if match:
rate.rooms_left = int(match.group(1))
hotels.append(hotel)
rates.append(rate)
except Exception as e:
logger.warning(f"Error extracting hotel data: {e}")
continue
return hotels, rates
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name
check_in: Check-in date
check_out: Check-out date (check_in + 1 for single night rate)
adults: Number of adults
pages: Number of result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
all_hotels = []
all_rates = []
seen_hotel_ids = set()
context = None
page = None
try:
context = await self._create_context()
page = await context.new_page()
for page_num in range(pages):
# Random delay between pages (3-7 seconds)
if page_num > 0:
delay = random.uniform(3, 7)
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
await asyncio.sleep(delay)
# Build URL with offset for pagination (25 results per page)
url = self._build_search_url(
location, check_in, check_out, adults,
offset=page_num * 25
)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='networkidle', timeout=30000)
except Exception as e:
logger.warning(f"Page load timeout, continuing: {e}")
# Check for blocking
content = await page.content()
is_blocked, reason = self.detect_blocking(content)
if is_blocked:
logger.warning(f"Blocking detected: {reason}")
return ScraperResult(
success=False,
blocked=True,
block_reason=reason,
hotels=all_hotels,
rates=all_rates,
page_content_sample=content[:1000]
)
# Human-like scrolling
await self._human_like_scroll(page)
# Extract data
hotels, rates = await self._extract_search_results(page, check_in)
# Deduplicate by booking_com_id
for hotel, rate in zip(hotels, rates):
if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids:
seen_hotel_ids.add(hotel.booking_com_id)
all_hotels.append(hotel)
all_rates.append(rate)
logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique")
return ScraperResult(
success=True,
blocked=False,
hotels=all_hotels,
rates=all_rates
)
except Exception as e:
logger.error(f"Scrape error: {e}")
return ScraperResult(
success=False,
blocked=False,
error_message=str(e),
hotels=all_hotels,
rates=all_rates
)
finally:
if page:
await page.close()
if context:
await context.close()
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape individual hotel page for detailed rates.
Future expansion - placeholder for now.
Will extract available_qty from room dropdowns.
"""
# Not implemented in Phase 2a
logger.warning("scrape_hotel_page not yet implemented")
return ScraperResult(
success=False,
error_message="Hotel page scraping not yet implemented"
)
async def close(self):
"""Clean up browser resources."""
if self._browser:
await self._browser.close()
self._browser = None
if self._playwright:
await self._playwright.stop()
self._playwright = None