hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add Rate Monitor app — Booking.com + direct booking engine competitor rates

Browse source 
Combines Booking.com Playwright scraper (from forecasting), direct booking
engine scraper (ported from laptop-archive/guestline-monitor), and Newbook
own-hotel rates into one focused tool. Four views: Bookability, Market View
(with price index badges + direct rate sub-rows), Direct Rates (per-competitor
room breakdown, min-stay flags, hotel config/discovery), Rate Analysis
(advance purchase curve, DOW chart, rate timeline, comparison table).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 12:06:30 +00:00
commit e05054172f
50 changed files with 11860 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

401
backend/services/scraper_backends/playwright_local.py Normal file
View file

@ -0,0 +1,401 @@
"""
Local Playwright backend for booking.com scraping.
Uses Playwright with Chromium to scrape search results.
No proxy - direct connection. Suitable for low-volume scraping.
"""
import asyncio
import logging
import random
import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import (
ScraperBackend,
ScraperResult,
HotelData,
RateData,
AvailabilityStatus
)
logger = logging.getLogger(__name__)
class PlaywrightLocalBackend(ScraperBackend):
"""
Local Playwright backend using Chromium.
Features:
- Rotates user agents
- Random delays between requests
- Mimics human scroll behavior
- Uses data-testid selectors for stability
"""
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
]
def __init__(self, proxy_config: dict = None):
"""
Initialize the backend.
Args:
proxy_config: Optional proxy configuration (for future use)
"""
self.proxy_config = proxy_config
self._playwright = None
self._browser: Optional[Browser] = None
async def _ensure_browser(self) -> Browser:
"""Ensure browser is running, start if needed."""
if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--no-sandbox',
'--disable-dev-shm-usage',
]
)
return self._browser
async def _create_context(self) -> BrowserContext:
"""Create a new browser context with random user agent."""
browser = await self._ensure_browser()
context = await browser.new_context(
user_agent=random.choice(self.USER_AGENTS),
viewport={'width': 1920, 'height': 1080},
locale='en-GB',
timezone_id='Europe/London',
)
return context
def _build_search_url(
self,
location: str,
check_in: date,
check_out: date,
adults: int,
offset: int = 0
) -> str:
"""Build booking.com search URL with parameters."""
params = {
'ss': location,
'checkin': check_in.isoformat(),
'checkout': check_out.isoformat(),
'group_adults': adults,
'no_rooms': 1,
'group_children': 0,
}
if offset > 0:
params['offset'] = offset
return f"https://www.booking.com/searchresults.en-gb.html?{urlencode(params)}"
def _parse_price(self, price_text: str) -> Optional[Decimal]:
"""Parse price from text like '£150' or 'GBP 150'."""
if not price_text:
return None
# Remove currency symbols and extract number
cleaned = re.sub(r'[£$€,\s]', '', price_text)
# Find first number (including decimals)
match = re.search(r'[\d,]+(?:\.\d{2})?', cleaned)
if match:
try:
return Decimal(match.group().replace(',', ''))
except InvalidOperation:
return None
return None
def _extract_hotel_id(self, url: str) -> Optional[str]:
"""Extract hotel ID from booking.com URL."""
if not url:
return None
# URL format: /hotel/gb/hotel-name.en-gb.html or ?dest_id=123
# Try to extract from URL path
match = re.search(r'/hotel/[a-z]{2}/([^/]+)\.', url)
if match:
return match.group(1)
# Try dest_id parameter
match = re.search(r'dest_id=(-?\d+)', url)
if match:
return match.group(1)
return None
async def _human_like_scroll(self, page: Page):
"""Simulate human-like scrolling behavior."""
# Scroll down in increments
for _ in range(3):
await page.mouse.wheel(0, random.randint(300, 600))
await asyncio.sleep(random.uniform(0.3, 0.8))
async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
"""Extract hotel and rate data from search results page."""
hotels = []
rates = []
# Wait for property cards - booking.com uses data-testid
try:
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
except Exception as e:
logger.warning(f"No property cards found: {e}")
return hotels, rates
# Get all property cards
cards = await page.query_selector_all('[data-testid="property-card"]')
logger.info(f"Found {len(cards)} property cards")
for card in cards:
try:
hotel = HotelData(booking_com_id='', name='')
rate = RateData(rate_date=rate_date)
# Hotel name
name_el = await card.query_selector('[data-testid="title"]')
if name_el:
hotel.name = (await name_el.inner_text()).strip()
if not hotel.name:
continue # Skip if no name found
# Hotel URL and ID
link_el = await card.query_selector('[data-testid="title-link"]')
if link_el:
hotel.booking_com_url = await link_el.get_attribute('href')
hotel.booking_com_id = self._extract_hotel_id(hotel.booking_com_url) or ''
rate.booking_com_id = hotel.booking_com_id
# Star rating - look for star icons or rating text
stars_el = await card.query_selector('[data-testid="rating-stars"]')
if stars_el:
stars_text = await stars_el.get_attribute('aria-label') or ''
match = re.search(r'(\d+)', stars_text)
if match:
hotel.star_rating = Decimal(match.group(1))
# Review score
score_el = await card.query_selector('[data-testid="review-score"]')
if score_el:
score_text = await score_el.inner_text()
match = re.search(r'([\d.]+)', score_text)
if match:
try:
hotel.review_score = Decimal(match.group(1))
except InvalidOperation:
pass
# Check for no availability message FIRST
no_avail_el = await card.query_selector('[data-testid="availability-message"]')
if no_avail_el:
avail_text = (await no_avail_el.inner_text()).lower()
if 'no availability' in avail_text or 'sold out' in avail_text:
rate.availability_status = AvailabilityStatus.SOLD_OUT
hotels.append(hotel)
rates.append(rate)
continue
# Price
price_el = await card.query_selector('[data-testid="price-and-discounted-price"]')
if not price_el:
# Try alternative selector
price_el = await card.query_selector('[data-testid="price"]')
if price_el:
price_text = await price_el.inner_text()
rate.rate_gross = self._parse_price(price_text)
if rate.rate_gross:
rate.availability_status = AvailabilityStatus.AVAILABLE
# Room type
room_el = await card.query_selector('[data-testid="recommended-units"]')
if room_el:
rate.room_type = (await room_el.inner_text()).strip()
# Rate option badges - try multiple selectors
# Breakfast included
breakfast_el = await card.query_selector('[data-testid="breakfast-included"]')
if not breakfast_el:
# Check text content for breakfast mentions
card_text = (await card.inner_text()).lower()
rate.breakfast_included = 'breakfast included' in card_text
else:
rate.breakfast_included = True
# Free cancellation
cancel_el = await card.query_selector('[data-testid="cancellation-policy"]')
if cancel_el:
cancel_text = (await cancel_el.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in cancel_text
else:
card_text = (await card.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in card_text
# No prepayment
prepay_el = await card.query_selector('[data-testid="no-prepayment"]')
if prepay_el:
rate.no_prepayment = True
else:
card_text = (await card.inner_text()).lower()
rate.no_prepayment = 'no prepayment' in card_text
# Rooms left / scarcity indicator
scarcity_el = await card.query_selector('[data-testid="availability-rate"]')
if scarcity_el:
scarcity_text = await scarcity_el.inner_text()
match = re.search(r'(\d+)\s*room', scarcity_text.lower())
if match:
rate.rooms_left = int(match.group(1))
hotels.append(hotel)
rates.append(rate)
except Exception as e:
logger.warning(f"Error extracting hotel data: {e}")
continue
return hotels, rates
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name
check_in: Check-in date
check_out: Check-out date (check_in + 1 for single night rate)
adults: Number of adults
pages: Number of result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
all_hotels = []
all_rates = []
seen_hotel_ids = set()
context = None
page = None
try:
context = await self._create_context()
page = await context.new_page()
for page_num in range(pages):
# Random delay between pages (3-7 seconds)
if page_num > 0:
delay = random.uniform(3, 7)
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
await asyncio.sleep(delay)
# Build URL with offset for pagination (25 results per page)
url = self._build_search_url(
location, check_in, check_out, adults,
offset=page_num * 25
)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='networkidle', timeout=30000)
except Exception as e:
logger.warning(f"Page load timeout, continuing: {e}")
# Check for blocking
content = await page.content()
is_blocked, reason = self.detect_blocking(content)
if is_blocked:
logger.warning(f"Blocking detected: {reason}")
return ScraperResult(
success=False,
blocked=True,
block_reason=reason,
hotels=all_hotels,
rates=all_rates,
page_content_sample=content[:1000]
)
# Human-like scrolling
await self._human_like_scroll(page)
# Extract data
hotels, rates = await self._extract_search_results(page, check_in)
# Deduplicate by booking_com_id
for hotel, rate in zip(hotels, rates):
if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids:
seen_hotel_ids.add(hotel.booking_com_id)
all_hotels.append(hotel)
all_rates.append(rate)
logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique")
return ScraperResult(
success=True,
blocked=False,
hotels=all_hotels,
rates=all_rates
)
except Exception as e:
logger.error(f"Scrape error: {e}")
return ScraperResult(
success=False,
blocked=False,
error_message=str(e),
hotels=all_hotels,
rates=all_rates
)
finally:
if page:
await page.close()
if context:
await context.close()
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape individual hotel page for detailed rates.
Future expansion - placeholder for now.
Will extract available_qty from room dropdowns.
"""
# Not implemented in Phase 2a
logger.warning("scrape_hotel_page not yet implemented")
return ScraperResult(
success=False,
error_message="Hotel page scraping not yet implemented"
)
async def close(self):
"""Clean up browser resources."""
if self._browser:
await self._browser.close()
self._browser = None
if self._playwright:
await self._playwright.stop()
self._playwright = None