hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add Rate Monitor app — Booking.com + direct booking engine competitor rates

Browse source 
Combines Booking.com Playwright scraper (from forecasting), direct booking
engine scraper (ported from laptop-archive/guestline-monitor), and Newbook
own-hotel rates into one focused tool. Four views: Bookability, Market View
(with price index badges + direct rate sub-rows), Direct Rates (per-competitor
room breakdown, min-stay flags, hotel config/discovery), Rate Analysis
(advance purchase curve, DOW chart, rate timeline, comparison table).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 12:06:30 +00:00
commit e05054172f
50 changed files with 11860 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
backend/services/__init__.py Normal file
View file

829
backend/services/booking_scraper.py Normal file
View file

@ -0,0 +1,829 @@
"""
Booking.com Rate Scraper Service
Main service for scraping competitor rates from booking.com.
Uses pluggable backends (Playwright local, proxy, Apify) via factory pattern.
Features:
- Location-based search (1 query = 40+ hotels)
- Hotel discovery and tier management
- Rate extraction with availability status
- Anti-scrape detection and pause/resume
"""
import logging
import uuid
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import List, Optional, Dict, Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
logger = logging.getLogger(__name__)
def get_scraper_backend(db: Session) -> ScraperBackend:
"""
Factory to get configured scraper backend.
Reads backend type from system_config and returns appropriate instance.
"""
# Get backend configuration
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'")
).fetchone()
backend_type = result.config_value if result and result.config_value else 'playwright_local'
if backend_type == 'playwright_local':
return PlaywrightLocalBackend()
elif backend_type == 'playwright_proxy':
# Get proxy config
proxy_result = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password')
""")
)
proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()}
return PlaywrightLocalBackend(proxy_config=proxy_config)
elif backend_type == 'apify':
# Future: Apify backend
raise NotImplementedError("Apify backend not yet implemented")
else:
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
return PlaywrightLocalBackend()
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
"""Get the active scrape location configuration."""
result = db.execute(
text("""
SELECT id, location_name, location_search_url, pages_to_scrape, adults
FROM booking_scrape_config
WHERE is_active = TRUE
ORDER BY id
LIMIT 1
""")
).fetchone()
if not result:
return None
return {
'id': result.id,
'location_name': result.location_name,
'location_search_url': result.location_search_url,
'pages_to_scrape': result.pages_to_scrape or 2,
'adults': result.adults or 2,
}
async def is_scraper_paused(db: Session) -> bool:
"""Check if scraper is currently paused due to blocking."""
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
).fetchone()
if not result or result.config_value != 'true':
return False
# Check if pause period has expired
pause_until_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'")
).fetchone()
if pause_until_result and pause_until_result.config_value:
try:
pause_until = datetime.fromisoformat(pause_until_result.config_value)
if datetime.now() >= pause_until:
# Pause expired, reset
db.execute(
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
)
db.commit()
return False
except ValueError:
pass
return True
async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
"""Set scraper pause status."""
db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
{'val': 'true' if paused else 'false'}
)
if paused:
pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
{'val': pause_until}
)
db.commit()
def save_hotel(db: Session, hotel: HotelData) -> int:
"""
Save or update a hotel in the database.
Returns the hotel's database ID.
"""
# Check if hotel exists
existing = db.execute(
text("SELECT id FROM booking_com_hotels WHERE booking_com_id = :bid"),
{'bid': hotel.booking_com_id}
).fetchone()
if existing:
# Update last_seen_at and any changed fields
db.execute(
text("""
UPDATE booking_com_hotels SET
name = COALESCE(:name, name),
booking_com_url = COALESCE(:url, booking_com_url),
star_rating = COALESCE(:stars, star_rating),
review_score = COALESCE(:score, review_score),
review_count = COALESCE(:count, review_count),
last_seen_at = NOW()
WHERE booking_com_id = :bid
"""),
{
'bid': hotel.booking_com_id,
'name': hotel.name,
'url': hotel.booking_com_url,
'stars': float(hotel.star_rating) if hotel.star_rating else None,
'score': float(hotel.review_score) if hotel.review_score else None,
'count': hotel.review_count,
}
)
return existing.id
else:
# Insert new hotel (default tier is 'market')
result = db.execute(
text("""
INSERT INTO booking_com_hotels
(booking_com_id, name, booking_com_url, star_rating, review_score, review_count, tier)
VALUES (:bid, :name, :url, :stars, :score, :count, 'market')
RETURNING id
"""),
{
'bid': hotel.booking_com_id,
'name': hotel.name,
'url': hotel.booking_com_url,
'stars': float(hotel.star_rating) if hotel.star_rating else None,
'score': float(hotel.review_score) if hotel.review_score else None,
'count': hotel.review_count,
}
)
return result.fetchone().id
def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
"""Save a rate to the database."""
db.execute(
text("""
INSERT INTO booking_com_rates
(hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id)
VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
:breakfast, :cancel, :prepay, :rooms_left, :batch_id)
"""),
{
'hotel_id': hotel_id,
'rate_date': rate.rate_date,
'status': rate.availability_status.value,
'rate': float(rate.rate_gross) if rate.rate_gross else None,
'currency': rate.currency,
'room_type': rate.room_type,
'breakfast': rate.breakfast_included,
'cancel': rate.free_cancellation,
'prepay': rate.no_prepayment,
'rooms_left': rate.rooms_left,
'batch_id': str(batch_id),
}
)
def create_scrape_batch(db: Session, scrape_type: str) -> uuid.UUID:
"""Create a new scrape batch log entry."""
batch_id = uuid.uuid4()
db.execute(
text("""
INSERT INTO booking_scrape_log
(batch_id, scrape_type, started_at, status)
VALUES (:batch_id, :scrape_type, NOW(), 'running')
"""),
{'batch_id': str(batch_id), 'scrape_type': scrape_type}
)
db.commit()
return batch_id
def update_scrape_batch(
db: Session,
batch_id: uuid.UUID,
status: str,
hotels_found: int = 0,
rates_scraped: int = 0,
error_message: str = None,
blocked: bool = False
):
"""Update scrape batch log with results."""
db.execute(
text("""
UPDATE booking_scrape_log SET
completed_at = CASE WHEN :status IN ('completed', 'failed', 'blocked') THEN NOW() ELSE NULL END,
status = :status,
hotels_found = :hotels,
rates_scraped = :rates,
error_message = :error,
blocked_at = CASE WHEN :blocked THEN NOW() ELSE NULL END,
resume_after = CASE WHEN :blocked THEN NOW() + INTERVAL '2 hours' ELSE NULL END
WHERE batch_id = :batch_id
"""),
{
'batch_id': str(batch_id),
'status': status,
'hotels': hotels_found,
'rates': rates_scraped,
'error': error_message,
'blocked': blocked,
}
)
db.commit()
def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
"""
Mark any 'running' scrape batches as 'failed' if they've been running
longer than max_age_minutes. This handles orphaned batches from
container restarts or crashes.
"""
result = db.execute(
text("""
UPDATE booking_scrape_log SET
status = 'failed',
completed_at = NOW(),
error_message = 'Interrupted (container restart or timeout)'
WHERE status = 'running'
AND started_at < NOW() - INTERVAL ':mins minutes'
RETURNING batch_id
""".replace(':mins', str(int(max_age_minutes))))
)
cleaned = result.fetchall()
db.commit()
if cleaned:
logger.info(f"Cleaned up {len(cleaned)} stale running scrape batch(es)")
return len(cleaned)
async def scrape_date(
db: Session,
rate_date: date,
backend: ScraperBackend,
config: Dict[str, Any],
batch_id: uuid.UUID
) -> Dict[str, Any]:
"""
Scrape rates for a single date.
Args:
db: Database session
rate_date: Date to scrape rates for
backend: Scraper backend instance
config: Scrape configuration
batch_id: Current batch ID
Returns:
Dict with 'success', 'blocked', 'hotels_count', 'rates_count'
"""
check_in = rate_date
check_out = rate_date + timedelta(days=1) # Single night
result = await backend.scrape_location_search(
location=config['location_name'],
check_in=check_in,
check_out=check_out,
adults=config['adults'],
pages=config['pages_to_scrape']
)
if result.blocked:
return {
'success': False,
'blocked': True,
'block_reason': result.block_reason,
'hotels_count': 0,
'rates_count': 0,
}
if not result.success:
return {
'success': False,
'blocked': False,
'error': result.error_message,
'hotels_count': 0,
'rates_count': 0,
}
# Save hotels and rates
hotels_saved = 0
rates_saved = 0
for hotel, rate in zip(result.hotels, result.rates):
if not hotel.booking_com_id:
continue
try:
hotel_id = save_hotel(db, hotel)
save_rate(db, rate, hotel_id, batch_id)
hotels_saved += 1
rates_saved += 1
except Exception as e:
logger.warning(f"Error saving hotel/rate: {e}")
continue
db.commit()
return {
'success': True,
'blocked': False,
'hotels_count': hotels_saved,
'rates_count': rates_saved,
}
async def run_manual_scrape(
db: Session,
from_date: date,
to_date: date = None
) -> Dict[str, Any]:
"""
Run a manual scrape for testing/on-demand use.
Args:
db: Database session
from_date: Start date
to_date: End date (defaults to from_date for single day)
Returns:
Dict with scrape results summary
"""
if to_date is None:
to_date = from_date
# Check if paused
if await is_scraper_paused(db):
return {
'success': False,
'error': 'Scraper is currently paused due to blocking. Try again later.',
}
# Get config
config = get_scrape_config(db)
if not config:
return {
'success': False,
'error': 'No scrape location configured. Add a location in settings.',
}
# Create batch
batch_id = create_scrape_batch(db, 'manual')
# Get backend
backend = get_scraper_backend(db)
total_hotels = 0
total_rates = 0
dates_completed = 0
dates_failed = 0
try:
current_date = from_date
while current_date <= to_date:
logger.info(f"Scraping date: {current_date}")
result = await scrape_date(db, current_date, backend, config, batch_id)
if result['blocked']:
# Blocking detected - pause and exit
await set_scraper_paused(db, True, hours=2)
update_scrape_batch(
db, batch_id,
status='blocked',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
blocked=True
)
return {
'success': False,
'blocked': True,
'block_reason': result.get('block_reason'),
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
if result['success']:
total_hotels += result['hotels_count']
total_rates += result['rates_count']
dates_completed += 1
else:
dates_failed += 1
logger.warning(f"Failed to scrape {current_date}: {result.get('error')}")
current_date += timedelta(days=1)
# Update batch as completed
update_scrape_batch(
db, batch_id,
status='completed',
hotels_found=total_hotels,
rates_scraped=total_rates
)
return {
'success': True,
'blocked': False,
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
except Exception as e:
logger.error(f"Scrape error: {e}")
update_scrape_batch(
db, batch_id,
status='failed',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=str(e)
)
return {
'success': False,
'error': str(e),
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
finally:
await backend.close()
# ============================================
# QUEUE MANAGEMENT
# ============================================
def populate_queue(db: Session, dates: List[date], priorities: Dict[date, int] = None):
"""
Add dates to the scrape queue, skipping any already pending/processing.
Args:
db: Database session
dates: Dates to add to the queue
priorities: Optional priority map (higher = scraped first). Default: 0
"""
if not dates:
return 0
added = 0
for rate_date in dates:
priority = (priorities or {}).get(rate_date, 0)
try:
db.execute(
text("""
INSERT INTO booking_scrape_queue (rate_date, status, priority)
VALUES (:rate_date, 'pending', :priority)
ON CONFLICT (rate_date, status) DO UPDATE SET
priority = GREATEST(booking_scrape_queue.priority, :priority)
"""),
{'rate_date': rate_date, 'priority': priority}
)
added += 1
except Exception:
# Ignore duplicates or constraint issues
pass
db.commit()
logger.info(f"Queue: added/updated {added} dates")
return added
def get_pending_queue_items(db: Session, limit: int = 50) -> List[Dict[str, Any]]:
"""Get pending queue items ordered by priority (highest first), then date."""
result = db.execute(
text("""
SELECT id, rate_date, priority, attempts, max_attempts
FROM booking_scrape_queue
WHERE status = 'pending' AND attempts < max_attempts
ORDER BY priority DESC, rate_date ASC
LIMIT :limit
"""),
{'limit': limit}
)
return [dict(row._mapping) for row in result.fetchall()]
def mark_queue_item(db: Session, queue_id: int, status: str, error_message: str = None):
"""Update a queue item's status."""
if status == 'completed':
db.execute(
text("""
UPDATE booking_scrape_queue SET
status = 'completed',
completed_at = NOW(),
last_attempt_at = NOW(),
attempts = attempts + 1
WHERE id = :id
"""),
{'id': queue_id}
)
elif status == 'failed':
db.execute(
text("""
UPDATE booking_scrape_queue SET
status = CASE
WHEN attempts + 1 >= max_attempts THEN 'failed'
ELSE 'pending'
END,
last_attempt_at = NOW(),
attempts = attempts + 1,
error_message = :error
WHERE id = :id
"""),
{'id': queue_id, 'error': error_message}
)
db.commit()
def clear_old_queue_items(db: Session, days: int = 7):
"""Remove completed/failed queue items older than N days."""
db.execute(
text("""
DELETE FROM booking_scrape_queue
WHERE status IN ('completed', 'failed')
AND created_at < NOW() - INTERVAL ':days days'
""".replace(':days', str(int(days))))
)
db.commit()
async def process_queue(db: Session) -> Dict[str, Any]:
"""
Process pending items from the scrape queue.
Picks up pending items in priority order, scrapes each date,
and handles blocking/retries.
Returns:
Dict with processing results
"""
# Check if paused
if await is_scraper_paused(db):
return {
'success': False,
'error': 'Scraper is currently paused due to blocking.',
}
# Get config
config = get_scrape_config(db)
if not config:
return {
'success': False,
'error': 'No scrape location configured.',
}
# Get pending items
items = get_pending_queue_items(db, limit=200)
if not items:
return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'}
# Create batch
batch_id = create_scrape_batch(db, 'scheduled')
# Update batch with queue count
db.execute(
text("UPDATE booking_scrape_log SET dates_queued = :count WHERE batch_id = :bid"),
{'count': len(items), 'bid': str(batch_id)}
)
db.commit()
# Get backend
backend = get_scraper_backend(db)
total_hotels = 0
total_rates = 0
dates_completed = 0
dates_failed = 0
try:
for item in items:
rate_date = item['rate_date']
queue_id = item['id']
logger.info(f"Queue processing: {rate_date} (priority={item['priority']}, attempt={item['attempts']+1})")
result = await scrape_date(db, rate_date, backend, config, batch_id)
if result['blocked']:
# Mark this item as failed, pause, and stop
mark_queue_item(db, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
await set_scraper_paused(db, True, hours=2)
update_scrape_batch(
db, batch_id,
status='blocked',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
blocked=True
)
# Update dates counters
db.execute(
text("""
UPDATE booking_scrape_log SET
dates_completed = :completed,
dates_failed = :failed
WHERE batch_id = :bid
"""),
{'completed': dates_completed, 'failed': dates_failed + 1, 'bid': str(batch_id)}
)
db.commit()
return {
'success': False,
'blocked': True,
'block_reason': result.get('block_reason'),
'dates_completed': dates_completed,
'dates_failed': dates_failed + 1,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
if result['success']:
mark_queue_item(db, queue_id, 'completed')
total_hotels += result['hotels_count']
total_rates += result['rates_count']
dates_completed += 1
else:
mark_queue_item(db, queue_id, 'failed', result.get('error'))
dates_failed += 1
logger.warning(f"Queue: failed to scrape {rate_date}: {result.get('error')}")
# Update batch as completed
update_scrape_batch(
db, batch_id,
status='completed',
hotels_found=total_hotels,
rates_scraped=total_rates
)
db.execute(
text("""
UPDATE booking_scrape_log SET
dates_completed = :completed,
dates_failed = :failed
WHERE batch_id = :bid
"""),
{'completed': dates_completed, 'failed': dates_failed, 'bid': str(batch_id)}
)
db.commit()
return {
'success': True,
'blocked': False,
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
except Exception as e:
logger.error(f"Queue processing error: {e}")
update_scrape_batch(
db, batch_id,
status='failed',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=str(e)
)
return {
'success': False,
'error': str(e),
'dates_completed': dates_completed,
'dates_failed': dates_failed,
}
finally:
await backend.close()
def get_competitor_matrix(
db: Session,
from_date: date,
to_date: date,
include_market: bool = False
) -> List[Dict[str, Any]]:
"""
Get rate comparison matrix for competitors.
Args:
db: Database session
from_date: Start date
to_date: End date
include_market: Include market tier hotels
Returns:
List of rate records for matrix display
"""
tier_filter = "h.tier IN ('own', 'competitor')"
if include_market:
tier_filter = "h.tier IN ('own', 'competitor', 'market')"
result = db.execute(
text(f"""
SELECT
r.rate_date,
h.id AS hotel_id,
h.name AS hotel_name,
h.tier,
h.display_order,
h.star_rating,
h.review_score,
r.availability_status,
r.rate_gross,
r.room_type,
r.breakfast_included,
r.free_cancellation,
r.no_prepayment,
r.rooms_left,
r.scraped_at
FROM booking_latest_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE {tier_filter}
AND h.is_active = TRUE
AND r.rate_date BETWEEN :from_date AND :to_date
ORDER BY r.rate_date, h.display_order, h.name
"""),
{'from_date': from_date, 'to_date': to_date}
)
return [dict(row._mapping) for row in result.fetchall()]
def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
"""
Get list of discovered hotels.
Args:
db: Database session
tier: Filter by tier ('own', 'competitor', 'market') or None for all
Returns:
List of hotel records
"""
where_clause = "WHERE is_active = TRUE"
if tier:
where_clause += f" AND tier = '{tier}'"
result = db.execute(
text(f"""
SELECT
id, booking_com_id, name, booking_com_url,
star_rating, review_score, review_count,
tier, display_order, notes,
first_seen_at, last_seen_at
FROM booking_com_hotels
{where_clause}
ORDER BY display_order, name
""")
)
return [dict(row._mapping) for row in result.fetchall()]
def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
"""Update a hotel's tier and display order."""
if tier not in ('own', 'competitor', 'market'):
raise ValueError(f"Invalid tier: {tier}")
params = {'hotel_id': hotel_id, 'tier': tier}
set_clause = "tier = :tier"
if display_order is not None:
set_clause += ", display_order = :order"
params['order'] = display_order
db.execute(
text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
params
)
db.commit()

100
backend/services/central_settings.py Normal file
View file

@ -0,0 +1,100 @@
"""
Client for the stack's central Settings service.
NewBook credentials are managed once in the Settings app (LXC 116) and
fetched live by every app — the same pattern as cashup / room-planner /
maintenance (see their lib/newbook.js). Falls back to None if the service
is unreachable so callers can fall back to app-local config.
"""
import logging
import os
import time
from typing import Optional
import httpx
logger = logging.getLogger(__name__)
SETTINGS_URL = os.getenv("SETTINGS_URL", "")
SETTINGS_SECRET = os.getenv("SETTINGS_SECRET", "")
_CACHE_TTL = 60 # seconds — credentials change rarely; avoid hammering the service
_cache: dict = {}
async def get_integration(name: str) -> Optional[dict]:
"""
Fetch integration config (e.g. 'newbook') from the central Settings
service. Returns the config dict, or None if unavailable/unconfigured.
"""
if not SETTINGS_URL or not SETTINGS_SECRET:
return None
cached = _cache.get(name)
if cached and (time.monotonic() - cached[0]) < _CACHE_TTL:
return cached[1]
url = f"{SETTINGS_URL}/settings/api/internal/integration/{name}"
try:
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(
url, headers={"Authorization": f"Bearer {SETTINGS_SECRET}"}
)
resp.raise_for_status()
data = resp.json()
_cache[name] = (time.monotonic(), data)
return data
except Exception as e:
logger.warning(f"Central settings fetch failed for '{name}': {e}")
return None
def _extract_newbook(s: Optional[dict]) -> Optional[dict]:
if not s:
return None
creds = {
"api_key": s.get("api_key") or "",
"username": s.get("username") or "",
"password": s.get("password") or "",
"region": s.get("region") or "eu",
}
# Only usable if the essential fields are present
if not (creds["api_key"] and creds["username"] and creds["password"]):
return None
return creds
async def get_newbook_credentials() -> Optional[dict]:
"""
Returns {'api_key', 'username', 'password', 'region'} from central
settings, or None if not available (caller should fall back).
"""
return _extract_newbook(await get_integration("newbook"))
def get_integration_sync(name: str) -> Optional[dict]:
"""Blocking variant of get_integration for sync job contexts."""
if not SETTINGS_URL or not SETTINGS_SECRET:
return None
cached = _cache.get(name)
if cached and (time.monotonic() - cached[0]) < _CACHE_TTL:
return cached[1]
url = f"{SETTINGS_URL}/settings/api/internal/integration/{name}"
try:
resp = httpx.get(
url, headers={"Authorization": f"Bearer {SETTINGS_SECRET}"}, timeout=5.0
)
resp.raise_for_status()
data = resp.json()
_cache[name] = (time.monotonic(), data)
return data
except Exception as e:
logger.warning(f"Central settings fetch failed for '{name}': {e}")
return None
def get_newbook_credentials_sync() -> Optional[dict]:
"""Blocking variant of get_newbook_credentials for sync job contexts."""
return _extract_newbook(get_integration_sync("newbook"))

28
backend/services/direct_profiles/__init__.py Normal file
View file

@ -0,0 +1,28 @@
from .guestline import GuestlineProfile
from .newbook_scrape import NewbookScrapeProfile
from .travelclick import TravelClickProfile
from .directbook import DirectBookProfile
from .base import BaseProfile
PROFILES = {
"guestline": GuestlineProfile,
"newbook_scrape": NewbookScrapeProfile,
"travelclick": TravelClickProfile,
"directbook": DirectBookProfile,
}
def get_profile(name: str) -> BaseProfile:
cls = PROFILES.get(name)
if not cls:
raise ValueError(f"Unknown engine profile: {name}")
return cls()
def detect_profile(url: str) -> dict | None:
"""Given a booking URL, return suggested profile name and extracted params."""
for name, cls in PROFILES.items():
result = cls.detect(url)
if result is not None:
return {"profile": name, **result}
return None

25
backend/services/direct_profiles/base.py Normal file
View file

@ -0,0 +1,25 @@
from abc import ABC, abstractmethod
class BaseProfile(ABC):
name: str = ""
label: str = ""
# Fields required to configure this engine, shown in the add-hotel form
# Each entry: {"key": "hotel_id", "label": "Hotel ID", "help": "e.g. THREEWAYS"}
required_params: list[dict] = []
@classmethod
@abstractmethod
def detect(cls, url: str) -> dict | None:
"""Return extracted params dict if URL matches this engine, else None."""
@abstractmethod
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
"""Return list of bookable arrival date strings YYYY-MM-DD."""
@abstractmethod
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
"""Return list of room/rate dicts for the stay.
Each dict must have: roomId, rateId, availability, prices[{amountBeforeTax, amountAfterTax}], currencyCode
prices list has one entry per night when nights > 1.
"""

175
backend/services/direct_profiles/directbook.py Normal file
View file

@ -0,0 +1,175 @@
import json
import re
from datetime import date, timedelta
from urllib.parse import quote
from .base import BaseProfile
API_BASE = "https://direct-book.com"
SETTINGS_HASH = "52d786f5c45232c8c16022bc3af6dab2e1994f4919b953afafe188095125e9b6"
QUOTESETS_HASH = "1012a6203854357e44786380240eefad6b2ad863aee6ba79748c81a851f29217"
ROOMTYPES_HASH = "8021345a2e1f993717b1960097489b456a6b2dc136990b6f919adba1b1fe2c1f"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
# Required to bypass Apollo CSRF protection on the /api/graphql endpoint
"Content-Type": "application/json",
}
def _graphql_url(operation: str, variables: dict, sha256: str) -> str:
return (
f"{API_BASE}/api/graphql"
f"?operationName={operation}"
f"&variables={_json_compact(variables)}"
f"&extensions={_json_compact({'persistedQuery': {'version': 1, 'sha256Hash': sha256}})}"
)
def _json_compact(obj) -> str:
return quote(json.dumps(obj, separators=(',', ':')), safe='')
async def _get_property_id(client, channel_code: str) -> str:
"""Fetch numeric propertyId from settings query."""
url = _graphql_url("settings", {"channelCode": channel_code}, SETTINGS_HASH)
r = await client.get(url, headers=HEADERS, timeout=20)
r.raise_for_status()
return str(r.json()["data"]["settings"]["uuid"])
class DirectBookProfile(BaseProfile):
name = "directbook"
label = "SiteMinder Direct Book"
required_params = [
{"key": "channel_code", "label": "Channel Code",
"help": "The property slug in the booking URL, e.g. 'grapevinestowdirect'"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
m = re.search(r'direct-book\.com/properties/([^/?#\s]+)', url)
if m:
return {"channel_code": m.group(1)}
return None
async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]:
channel_code = params["channel_code"]
url = _graphql_url(
"roomTypes",
{"channelCode": channel_code, "checkInDate": date.today().isoformat(),
"checkOutDate": (date.today() + timedelta(days=1)).isoformat(), "locale": "en"},
ROOMTYPES_HASH,
)
r = await client.get(url, headers=HEADERS, timeout=30)
r.raise_for_status()
room_types = r.json()["data"]["roomTypes"]
room_labels = {rt["uuid"]: rt["name"] for rt in room_types}
rate_labels = {}
for rt in room_types:
for rate in rt.get("rates", []):
rate_labels[rate["uuid"]] = rate["name"]
return room_labels, rate_labels
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
channel_code = params["channel_code"]
today = date.today()
# Fetch 12 months of availability in monthly chunks (API seems to accept wide ranges too)
end = today.replace(day=1) + timedelta(days=365)
# Build monthly windows to stay within API limits
available_dates: list[str] = []
current = today.replace(day=1)
while current <= end:
# Last day of the month
if current.month == 12:
month_end = current.replace(year=current.year + 1, month=1, day=1) - timedelta(days=1)
else:
month_end = current.replace(month=current.month + 1, day=1) - timedelta(days=1)
check_from = max(today, current).isoformat()
check_to = month_end.isoformat()
url = (
f"{API_BASE}/api/properties/{channel_code}/availability"
f"?checkInsFrom={check_from}&checkInsTo={check_to}"
)
try:
r = await client.get(url, headers=HEADERS, timeout=20)
r.raise_for_status()
for entry in r.json().get("result", []):
if entry.get("canCheckIn"):
d = entry["date"][:10]
if d >= today.isoformat():
available_dates.append(d)
except Exception:
pass
# Advance to next month
if current.month == 12:
current = current.replace(year=current.year + 1, month=1)
else:
current = current.replace(month=current.month + 1)
return sorted(set(available_dates))
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
channel_code = params["channel_code"]
# Get propertyId (numeric) — cache it on the params dict to avoid repeat fetches
if "property_id" not in params:
params["property_id"] = await _get_property_id(client, channel_code)
property_id = int(params["property_id"])
departure = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
url = _graphql_url(
"quoteSets",
{
"propertyId": property_id,
"promocode": "",
"checkInDate": arrival,
"checkOutDate": departure,
"adults": 2,
"children": 0,
"infants": 0,
"currencyFrom": "GBP",
"currencyTo": "GBP",
},
QUOTESETS_HASH,
)
r = await client.get(url, headers=HEADERS, timeout=20)
if r.status_code == 404:
return []
r.raise_for_status()
rooms = []
for qs in r.json()["data"].get("quoteSets", []):
room_id = str(qs["roomTypeId"])
for quote in qs.get("quotes", []):
rate_id = str(quote["roomRateId"])
price = quote["price"]["amount"]
available = quote.get("available", 0)
# Build per-night prices from breakdown if multi-night
breakdown = quote.get("breakdown", [])
if breakdown:
prices = [
{"amountBeforeTax": b["price"]["amount"], "amountAfterTax": b["price"]["amount"]}
for b in breakdown
]
else:
prices = [{"amountBeforeTax": price, "amountAfterTax": price}]
rooms.append({
"roomId": room_id,
"rateId": rate_id,
"availability": available,
"prices": prices,
"min_stay_nights": None,
"currencyCode": "GBP",
})
return rooms

49
backend/services/direct_profiles/guestline.py Normal file
View file

@ -0,0 +1,49 @@
import re
from datetime import date, timedelta
from .base import BaseProfile
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
}
class GuestlineProfile(BaseProfile):
name = "guestline"
label = "Guestline"
required_params = [
{"key": "hotel_id", "label": "Hotel ID", "help": "e.g. THREEWAYS — from the booking URL ?hotel= parameter"},
{"key": "collection_id", "label": "Collection ID", "help": "e.g. MT — the path segment before /availability in the booking URL"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
# Matches: https://booking.eu.guestline.app/MT/availability?hotel=THREEWAYS
m = re.search(r'booking\.(?:eu\.)?guestline\.app/([^/?\s]+)/availability\?hotel=([^&\s]+)', url)
if m:
return {"collection_id": m.group(1), "hotel_id": m.group(2)}
return None
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
hotel_id = params["hotel_id"]
today = date.today()
url = f"https://booking.eu.guestline.app/api/availabilities/{hotel_id}/arrivals"
r = await client.get(url, params={
"month": today.month, "year": today.year,
"adults": 2, "children": 0, "count": 12,
}, headers=HEADERS, timeout=20)
r.raise_for_status()
return [a["date"] for a in r.json().get("arrivals", [])]
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
hotel_id = params["hotel_id"]
collection_id = params.get("collection_id", "MT")
dep = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}/enhanced"
r = await client.get(url, params={
"arrival": arrival, "departure": dep, "adults": 2, "children": 0,
}, headers=HEADERS, timeout=20)
if r.status_code == 404:
return []
r.raise_for_status()
return r.json().get("availabilities", {}).get("rooms", [])

235
backend/services/direct_profiles/newbook_scrape.py Normal file
View file

@ -0,0 +1,235 @@
import json
import re
from datetime import date, timedelta
from .base import BaseProfile
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "*/*",
"X-Requested-With": "XMLHttpRequest",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
}
BASE_URL = "https://bookingseu.newbook.cloud"
def _base_params(slug: str, arrival: str, departure: str, nights: int) -> dict:
return {
"REMOTE_ADDR": "1.1.1.1",
"force_booking_channel_id": "",
"HTTP_REFERER": f"bookingseu.newbook.cloud/{slug}/index.php",
"discount_total_display": "0",
"force_category_id[]": "uK0ip@c7ty%8bQ#2i",
"force_category_type_id[]": "uK0ip@c7ty%8bQ#2i",
"no_billing_booking": "0",
"force_tariff_type_id[]": "uK0ip@c7ty%8bQ#2i",
"discount_id": "null",
"facebook_user_id": "",
"category_type_id": "",
"owner_occupied_booking_id": "",
"discount_code": "",
"booking_action": "",
"available_from": arrival,
"available_to": departure,
"nights": str(nights),
"adults": "2",
"children": "0",
"infants": "0",
"promo_code": "",
"language": "EN",
}
def _fmt_date(d: date) -> str:
"""Format date as NewBook expects: 'Mon 4 Jul 2026'"""
return d.strftime("%a %-d %b %Y")
def _parse_chart_html(html: str) -> tuple[dict[str, float], dict[str, str], dict[str, str], list[dict]]:
"""
Parse an availability_chart_responsive HTML response.
Returns:
counts: {cat_id: float} — room counts from category_sites_available JS var
cat_names: {cat_id: str} — friendly category names e.g. "Executive Double"
rate_names: {rate_id: str} — friendly tariff names e.g. "DIRECT B&B FLEX"
rooms: list of room/rate dicts compatible with base scraper format
"""
# Room counts from embedded JS
counts: dict[str, float] = {}
m = re.search(r'category_sites_available\s*=\s*(\{[^;]+\})', html)
if m:
try:
counts = {k: float(v) for k, v in json.loads(m.group(1)).items()}
except Exception:
pass
cat_names: dict[str, str] = {}
rate_names: dict[str, str] = {}
rooms = []
# Split by category box: offset="{cat_id}"
cat_blocks = re.split(r'<div[^>]+class="[^"]*newbook_online_category_box[^"]*"[^>]+offset="(\d+)"', html)
# cat_blocks: [pre, cat_id, block, cat_id, block, ...]
i = 1
while i < len(cat_blocks) - 1:
cat_id = cat_blocks[i]
block = cat_blocks[i + 1]
i += 2
avail = counts.get(cat_id, 0.0)
# Category friendly name — from category_name attr on any book button in this block
# e.g. category_name='Standard' or category_name='Executive Double'
cn_m = re.search(r"category_name='([^']+)'", block)
if not cn_m:
# Fallback: <h3><a ...>Name</a></h3>
cn_m = re.search(r'<h3>[^<]*<a[^>]*>([^<]+)</a>', block)
if cn_m:
cat_names[cat_id] = cn_m.group(1).strip()
# Split on tariff row boundaries
tariff_rows = re.split(r'class="[^"]*newbook_online_categories_tariff_type_rows[^"]*"', block)
for row in tariff_rows[1:]:
# Rate label
name_m = re.search(r'newbook_online_categories_tariff_type_label[^>]*>(.*?)</div>', row, re.DOTALL)
rate_label = re.sub(r'<[^>]+>', '', name_m.group(1)).strip() if name_m else ""
# Price
price_m = re.search(r'newbook_online_from_price_text[^>]*>£([\d.]+)<', row)
price = float(price_m.group(1)) if price_m else None
# Internal tariff type ID (stable across dates)
tid_m = re.search(r'tariff_type_id="(\d+)"', row)
rate_id = tid_m.group(1) if tid_m else rate_label
# Store rate label for this rate_id
if rate_id and rate_label:
rate_names[rate_id] = rate_label
# Min-stay: requires_date_change class + optional extend_nights attr
# If extend_nights present: min_stay = 1 + N; if absent: default to 2
min_stay = None
if 'requires_date_change' in row:
en_m = re.search(r'extend_nights="(\d+)"', row)
min_stay = 1 + int(en_m.group(1)) if en_m else 2
if rate_label and price is not None:
rooms.append({
"roomId": cat_id,
"rateId": rate_id,
"rateLabel": rate_label,
"availability": int(avail),
"prices": [{"amountBeforeTax": price, "amountAfterTax": price}],
"min_stay_nights": min_stay,
"currencyCode": "GBP",
})
return counts, cat_names, rate_names, rooms
class NewbookScrapeProfile(BaseProfile):
name = "newbook_scrape"
label = "NewBook (HTML scrape)"
required_params = [
{"key": "slug", "label": "Property Slug",
"help": "The path segment in the booking URL, e.g. 'numberfour' from bookingseu.newbook.cloud/numberfour/"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
m = re.search(r'bookingseu\.newbook\.cloud/([^/?#\s]+)', url)
if m and m.group(1) not in ('index.php',):
return {"slug": m.group(1)}
return None
async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]:
"""
Return ({cat_id: friendly_name}, {rate_id: friendly_name}) from a single chart call.
Used by discovery to pre-populate room_labels and rate_labels.
"""
slug = params["slug"]
today = date.today()
base = _base_params(slug, _fmt_date(today), _fmt_date(today + timedelta(days=1)), 1)
r = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive",
data=base, headers=HEADERS, timeout=30,
)
r.raise_for_status()
_, cat_names, rate_names, _ = _parse_chart_html(r.text)
return cat_names, rate_names
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
"""
Use the calendar endpoint to collect all available arrival dates
across all room types. One call per room type, union of available dates.
We first do a chart call to discover category IDs, then calendar per category.
"""
slug = params["slug"]
today = date.today()
arrival_str = _fmt_date(today)
departure_str = _fmt_date(today + timedelta(days=1))
# Step 1: chart call to discover category IDs and names
base = _base_params(slug, arrival_str, departure_str, 1)
r = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive",
data=base, headers=HEADERS, timeout=30,
)
r.raise_for_status()
html = r.text
cat_ids = re.findall(r'class="[^"]*newbook_online_category_box[^"]*"[^>]+offset="(\d+)"', html)
if not cat_ids:
return []
# Step 2: calendar call per category, collect available dates
available_dates: set[str] = set()
more_tariffs = {f"more_tariffs_{cid}": "1" for cid in cat_ids}
for cat_id in cat_ids:
cal_params = {
**base,
**more_tariffs,
"query": "newbook_calendar_initialise",
"calendar_category_id": cat_id,
}
try:
cr = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=data",
data=cal_params, headers=HEADERS, timeout=30,
)
cr.raise_for_status()
cal_data = cr.json()
cal_html = cal_data.get("calendar_display", "")
for dm in re.finditer(r'class="day available[^"]*"\s+data-date="(\d{4}-\d{2}-\d{2})"', cal_html):
available_dates.add(dm.group(1))
except Exception:
pass
return sorted(d for d in available_dates if d >= today.isoformat())
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
"""
POST to availability_chart_responsive for a specific date.
Returns list of room/rate dicts compatible with the base scraper format.
Also injects min_stay_nights onto each row.
"""
slug = params["slug"]
arr = date.fromisoformat(arrival)
dep = arr + timedelta(days=nights)
arr_str = _fmt_date(arr)
dep_str = _fmt_date(dep)
body = _base_params(slug, arr_str, dep_str, nights)
r = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive",
data=body, headers=HEADERS, timeout=30,
)
if r.status_code == 404:
return []
r.raise_for_status()
_, _, _, rooms = _parse_chart_html(r.text)
return rooms

166
backend/services/direct_profiles/travelclick.py Normal file
View file

@ -0,0 +1,166 @@
import re
import httpx
from datetime import date, timedelta
from .base import BaseProfile
API_BASE = "https://api.travelclick.com"
TOKEN_URL = f"{API_BASE}/oauth/token-referer?grant_type=client_credentials"
HEADERS_BASE = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
"Content-Type": "application/json",
}
async def _get_token(client, referer: str) -> str:
r = await client.post(TOKEN_URL, headers={**HEADERS_BASE, "Referer": referer}, timeout=20)
r.raise_for_status()
return r.json()["access_token"]
def _auth_headers(token: str, referer: str) -> dict:
return {**HEADERS_BASE, "Authorization": f"Bearer {token}", "Referer": referer}
class TravelClickProfile(BaseProfile):
name = "travelclick"
label = "TravelClick / Amadeus"
required_params = [
{"key": "hotel_code", "label": "Hotel Code",
"help": "Numeric hotel ID, e.g. 77346 — visible in the booking engine network requests"},
{"key": "booking_url", "label": "Booking URL Base",
"help": "e.g. https://reservations.bespokehotels.com/noelarmshotel/book/dates-of-stay"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
m = re.search(r'(https://reservations\.bespokehotels\.com/[^/]+/book/[^?#\s]+)', url)
if not m:
return None
booking_url = m.group(1)
# hotel_code is in inline JS as bookingEngineHotelId: '77346' on the booking page
hotel_code = ""
try:
r = httpx.get(booking_url, timeout=10, follow_redirects=True,
headers={"User-Agent": "Mozilla/5.0"})
hm = re.search(r'bookingEngineHotelId\s*:\s*[\'"](\d+)[\'"]', r.text)
if hm:
hotel_code = hm.group(1)
except Exception:
pass
return {"booking_url": booking_url, "hotel_code": hotel_code}
async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]:
hotel_code = params["hotel_code"]
referer = params["booking_url"]
token = await _get_token(client, referer)
r = await client.get(
f"{API_BASE}/be5-entity/v2/hotels/{hotel_code}/content",
params={"include": "roomtypes,rateplans", "lang": "EN_US"},
headers=_auth_headers(token, referer),
timeout=30,
)
r.raise_for_status()
data = r.json()
# roomtypes[].roomTypeId (numeric, used as roomId in avail) -> roomTypeName
room_labels = {
str(rt["roomTypeId"]): rt.get("roomTypeName", str(rt["roomTypeId"]))
for rt in data.get("roomtypes", [])
}
# ratePlans[].rateplanId (numeric, used as rateId in avail) -> rateplanName
rate_labels = {
str(rp["rateplanId"]): rp.get("rateplanName", str(rp["rateplanId"]))
for rp in data.get("ratePlans", [])
}
return room_labels, rate_labels
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
hotel_code = params["hotel_code"]
referer = params["booking_url"]
token = await _get_token(client, referer)
today = date.today()
end = today + timedelta(days=365)
body = {
"hotelCode": int(hotel_code),
"currency": "GBP",
"lang": "EN_US",
"dateIn": today.isoformat(),
"dateOut": end.isoformat(),
"multiRoomOccupancy": [{"adults": 2, "infant": 0, "children": 0}],
"bookerIdentifier": "",
"partnerIdentifier": "",
}
r = await client.post(
f"{API_BASE}/be5-shop/v1/hotel/{hotel_code}/basicavail/multi-room",
json=body,
headers=_auth_headers(token, referer),
timeout=30,
)
r.raise_for_status()
data = r.json()
return [
d["date"]
for d in data.get("dates", [])
if d.get("isAvailable") and d["date"] >= today.isoformat()
]
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
hotel_code = params["hotel_code"]
referer = params["booking_url"]
token = await _get_token(client, referer)
departure = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
body = {
"roomStay": {
"startDate": arrival,
"endDate": departure,
"guestCount": {"adults": 2, "infants": 0, "children": {"ages": None, "count": 0}},
"roomQuantity": 1,
"productSearchCriteria": {
"sortingPreference": "SORT_BY_ORDER",
"includeUnavailable": False,
},
},
"languageCode": "EN_US",
"disableLocaitonSharing": False,
"currencyCode": "GBP",
"tpaExtension": [],
"includeMemberRate": True,
"includeNightlyRates": True,
}
r = await client.post(
f"{API_BASE}/be5-shop/v2/hotels/{hotel_code}/avail",
json=body,
headers=_auth_headers(token, referer),
timeout=30,
)
if r.status_code == 404:
return []
r.raise_for_status()
data = r.json()
# Response: roomStays[0].roomtypes[].products[] (both Regular rates and Packages)
room_stay = (data.get("roomStays") or [{}])[0]
rooms = []
for rt in room_stay.get("roomtypes", []):
room_id = str(rt["roomtypeId"])
for product in rt.get("products", []):
rate_id = str(product["productId"])
nightly = product.get("nightlyRates", [])
if not nightly:
continue
avail = nightly[0].get("inventoryCount", 0)
prices = [
{"amountBeforeTax": n["amountBeforeTax"], "amountAfterTax": n.get("amountTotal", n["amountBeforeTax"])}
for n in nightly
]
rooms.append({
"roomId": room_id,
"rateId": rate_id,
"availability": avail,
"prices": prices,
"min_stay_nights": None, # min-stay comes from basicavail per date
"currencyCode": "GBP",
})
return rooms

295
backend/services/direct_scraper.py Normal file
View file

@ -0,0 +1,295 @@
"""
Direct booking engine scraper — adapted from guestline-monitor/app/scraper.py.
Replaces SQLite per-hotel DBs with PostgreSQL via SyncSessionLocal.
Logic (min-stay detection, discovery date sampling) is unchanged from the original.
"""
import asyncio
import logging
from datetime import datetime, timezone, date, timedelta
import httpx
from sqlalchemy import text
from database import SyncSessionLocal
from services.direct_profiles import get_profile
log = logging.getLogger(__name__)
REQUEST_DELAY = 10.0
DISCOVERY_DELAY = 2.0
# Track running discovery scrapes: hotel_id -> status dict
_discovery_status: dict[int, dict] = {}
def _discovery_dates() -> list[str]:
"""42 spread dates: one per DOW over 6 months, 6 weeks apart."""
dates = []
today = date.today()
for week_offset in range(6):
base = today + timedelta(weeks=week_offset * 4)
for dow in range(7):
days_ahead = (dow - base.weekday()) % 7
d = base + timedelta(days=days_ahead + 7)
iso = d.isoformat()
if iso not in dates:
dates.append(iso)
return sorted(dates)
async def run_discovery(hotel_id: int, profile_name: str, params: dict):
"""Sample ~42 spread dates to find all room/rate type IDs for a competitor hotel."""
_discovery_status[hotel_id] = {
"state": "running", "done": 0, "total": 0,
"found_rooms": [], "found_rates": [],
}
profile = get_profile(profile_name)
dates = _discovery_dates()
_discovery_status[hotel_id]["total"] = len(dates)
found_rooms: set[str] = set()
found_rates: set[str] = set()
async with httpx.AsyncClient() as client:
for i, arrival in enumerate(dates):
await asyncio.sleep(DISCOVERY_DELAY)
try:
rooms = await profile.fetch_night_rates(client, params, arrival)
for room in rooms:
found_rooms.add(room["roomId"])
found_rates.add(room["rateId"])
except Exception as e:
log.warning(f"Discovery hotel {hotel_id} {arrival}: {e}")
_discovery_status[hotel_id]["done"] = i + 1
_discovery_status[hotel_id]["found_rooms"] = sorted(found_rooms)
_discovery_status[hotel_id]["found_rates"] = sorted(found_rates)
# Fetch friendly names if the profile supports it
if hasattr(profile, "fetch_category_names"):
try:
cat_names, rate_names = await profile.fetch_category_names(client, params)
_discovery_status[hotel_id]["room_labels"] = cat_names
_discovery_status[hotel_id]["rate_labels"] = rate_names
# Merge into DB (existing user labels win)
db = SyncSessionLocal()
try:
row = db.execute(
text("SELECT room_labels, rate_labels FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
).mappings().fetchone()
if row:
import json
existing_rooms = row["room_labels"] or {}
existing_rates = row["rate_labels"] or {}
merged_rooms = {**cat_names, **existing_rooms}
merged_rates = {**rate_names, **existing_rates}
db.execute(
text("""UPDATE direct_competitor_hotels
SET room_labels = :rl, rate_labels = :ratel
WHERE id = :id"""),
{"rl": json.dumps(merged_rooms), "ratel": json.dumps(merged_rates), "id": hotel_id}
)
db.commit()
finally:
db.close()
except Exception as e:
log.warning(f"Discovery hotel {hotel_id}: could not fetch category names: {e}")
_discovery_status[hotel_id]["state"] = "complete"
log.info(f"Discovery complete hotel {hotel_id}: {len(found_rooms)} rooms, {len(found_rates)} rates")
def get_discovery_status(hotel_id: int) -> dict:
return _discovery_status.get(hotel_id, {"state": "idle"})
def run_scrape(hotel_id: int, profile_name: str, params: dict):
"""Full scrape run for one configured hotel. Writes to direct_rates + direct_scrape_runs."""
scraped_at = datetime.now(timezone.utc)
log.info(f"Direct scrape started for hotel {hotel_id}")
profile = get_profile(profile_name)
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at))
finally:
loop.close()
db = SyncSessionLocal()
try:
db.execute(
text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"),
{"ts": scraped_at, "id": hotel_id}
)
db.commit()
finally:
db.close()
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
async with httpx.AsyncClient() as client:
try:
arrival_dates = await profile.fetch_arrival_dates(client, params)
except Exception as e:
log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}")
return
log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates")
db = SyncSessionLocal()
try:
result = db.execute(
text("INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found) VALUES (:hid, :ts, :df) RETURNING id"),
{"hid": hotel_id, "ts": scraped_at, "df": len(arrival_dates)}
)
run_id = result.fetchone()[0]
db.commit()
prev_dates = {r[0].isoformat() for r in db.execute(
text("SELECT DISTINCT stay_date FROM direct_rates WHERE hotel_id = :hid AND stay_date >= :today"),
{"hid": hotel_id, "today": date.today()}
).fetchall()}
finally:
db.close()
arrival_set = set(arrival_dates)
missing_dates = sorted(prev_dates - arrival_set)
if missing_dates:
log.info(f"Hotel {hotel_id}: {len(missing_dates)} dates absent, checking min-stay")
# Min-stay check for missing dates
for fd in missing_dates:
fd_date = date.fromisoformat(fd)
windows = [
(fd_date - timedelta(days=1), fd_date + timedelta(days=1)),
(fd_date, fd_date + timedelta(days=2)),
]
min_stay_rooms = None
min_stay_other_night = None
await asyncio.sleep(REQUEST_DELAY)
for win_start, win_end in windows:
try:
rooms_2n = await profile.fetch_night_rates(client, params, win_start.isoformat(), nights=2)
if rooms_2n:
min_stay_rooms = rooms_2n
companion = win_start if win_start.isoformat() != fd else (win_start + timedelta(days=1))
min_stay_other_night = companion.isoformat()
log.info(f" Hotel {hotel_id} {fd}: min-stay detected")
break
except Exception as e:
log.warning(f" Hotel {hotel_id} {fd}: min-stay check {win_start}: {e}")
db = SyncSessionLocal()
try:
last_rows = db.execute(
text("""SELECT DISTINCT ON (room_id, rate_id)
room_id, rate_id, availability, price_excl, price_incl, currency
FROM direct_rates
WHERE hotel_id = :hid AND stay_date = :fd
ORDER BY room_id, rate_id, scraped_at DESC"""),
{"hid": hotel_id, "fd": fd}
).mappings().fetchall()
insert_rows = []
for r in last_rows:
if min_stay_rooms is not None:
match = next(
(m for m in min_stay_rooms
if m["roomId"] == r["room_id"] and m["rateId"] == r["rate_id"]
and m.get("prices")),
None
)
if match:
prices = match["prices"]
win_start_used = windows[0][0] if min_stay_other_night == windows[0][0].isoformat() else windows[1][0]
idx = 1 if win_start_used.isoformat() != fd else 0
if len(prices) > idx:
p = prices[idx]
insert_rows.append({
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
"room_id": r["room_id"], "rate_id": r["rate_id"],
"avail": match.get("availability", r["availability"]),
"pe": p["amountBeforeTax"], "pi": p["amountAfterTax"],
"cur": match.get("currencyCode", r["currency"]), "ms": 2
})
continue
insert_rows.append({
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
"room_id": r["room_id"], "rate_id": r["rate_id"],
"avail": r["availability"], "pe": r["price_excl"], "pi": r["price_incl"],
"cur": r["currency"], "ms": 2
})
else:
insert_rows.append({
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
"room_id": r["room_id"], "rate_id": r["rate_id"],
"avail": 0, "pe": r["price_excl"], "pi": r["price_incl"],
"cur": r["currency"], "ms": None
})
if insert_rows:
db.execute(
text("""INSERT INTO direct_rates
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
availability, price_excl, price_incl, currency, min_stay_nights)
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
:avail, :pe, :pi, :cur, :ms)"""),
insert_rows
)
db.commit()
status = "min-stay(2N)" if min_stay_rooms else "fully-booked"
log.info(f" Hotel {hotel_id} {fd}: recorded as {status}")
finally:
db.close()
# Scrape all bookable dates
rows_saved = 0
for i, arrival in enumerate(arrival_dates):
await asyncio.sleep(REQUEST_DELAY)
try:
rooms = await profile.fetch_night_rates(client, params, arrival)
if not rooms:
continue
insert_rows = [
{
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": arrival,
"room_id": room["roomId"], "rate_id": room["rateId"],
"avail": room["availability"],
"pe": room["prices"][0]["amountBeforeTax"],
"pi": room["prices"][0]["amountAfterTax"],
"cur": room.get("currencyCode", "GBP"),
"ms": room.get("min_stay_nights")
}
for room in rooms if room.get("prices")
]
if insert_rows:
db = SyncSessionLocal()
try:
db.execute(
text("""INSERT INTO direct_rates
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
availability, price_excl, price_incl, currency, min_stay_nights)
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
:avail, :pe, :pi, :cur, :ms)"""),
insert_rows
)
db.commit()
finally:
db.close()
rows_saved += len(insert_rows)
log.info(f" Hotel {hotel_id} {arrival}: {len(rooms)} combos [{i+1}/{len(arrival_dates)}]")
except Exception as e:
log.error(f" Hotel {hotel_id} {arrival}: ERROR - {e}")
db = SyncSessionLocal()
try:
db.execute(
text("UPDATE direct_scrape_runs SET rows_saved = :rs WHERE id = :id"),
{"rs": rows_saved, "id": run_id}
)
db.commit()
finally:
db.close()
log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved")

863
backend/services/newbook_rates_client.py Normal file
View file

@ -0,0 +1,863 @@
"""
Newbook Rates Client
Fetches current rack rates from Newbook API for revenue forecasting.
Uses the bookings_availability_pricing endpoint to simulate booking requests.
This client is READ-ONLY - it only queries available rates, never creates bookings.
"""
import os
import httpx
import asyncio
import logging
from datetime import date, timedelta
from decimal import Decimal
from typing import Optional, List, Dict
logger = logging.getLogger(__name__)
class NewbookRatesError(Exception):
"""Custom exception for Newbook rates API errors"""
pass
class NewbookRatesClient:
"""
Async client for fetching current rates from Newbook API.
Uses bookings_availability_pricing endpoint which simulates a booking request.
Handles minimum stay restrictions by extending the stay period when needed.
Rate limiting: ~100 requests/min, using 0.75s delay between requests
"""
BASE_URL = "https://api.newbook.cloud/rest"
def __init__(self, api_key: str = None, username: str = None, password: str = None,
region: str = None, vat_rate: Decimal = Decimal('0.20')):
self.api_key = api_key or os.getenv("NEWBOOK_API_KEY")
self.username = username or os.getenv("NEWBOOK_USERNAME")
self.password = password or os.getenv("NEWBOOK_PASSWORD")
self.region = region or os.getenv("NEWBOOK_REGION")
self.vat_rate = vat_rate
if not all([self.api_key, self.username, self.password, self.region]):
logger.warning("Newbook credentials not fully configured")
def _get_url(self, endpoint: str) -> str:
"""Get full URL for an endpoint"""
return f"{self.BASE_URL}/{endpoint}"
@classmethod
async def from_db(cls, db):
"""
Create client with credentials from the central Settings service
(stack-wide NewBook config), falling back to the app-local
system_config table. VAT rate stays app-local either way.
"""
from sqlalchemy import text
from services.central_settings import get_newbook_credentials
result = await db.execute(
text("SELECT config_key, config_value FROM system_config WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')")
)
rows = result.fetchall()
config = {row.config_key: row.config_value for row in rows}
vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20'))
central = await get_newbook_credentials()
if central:
return cls(**central, vat_rate=vat_rate)
return cls(
api_key=config.get('newbook_api_key'),
username=config.get('newbook_username'),
password=config.get('newbook_password'),
region=config.get('newbook_region'),
vat_rate=vat_rate
)
async def __aenter__(self):
self.client = httpx.AsyncClient(timeout=60.0)
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.client.aclose()
def _get_auth_payload(self) -> dict:
"""Get base authentication payload"""
return {
"api_key": self.api_key,
"region": self.region
}
async def get_category_rates(
self,
category_id: str,
from_date: date,
to_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> List[Dict]:
"""
Fetch current rates for a category over a date range.
Uses daily=true to get per-night rates. Handles minimum stay
restrictions by extending the period when needed.
Args:
category_id: Newbook category ID
from_date: Start date for rates
to_date: End date for rates (inclusive)
guests_adults: Number of adult guests (default 2)
guests_children: Number of child guests (default 0)
Returns:
List of dicts with {date, gross_rate, net_rate}
"""
rates = []
current_date = from_date
while current_date <= to_date:
try:
# Fetch rates for up to 7 days at a time to optimize API calls
batch_end = min(current_date + timedelta(days=6), to_date)
batch_rates = await self._fetch_rates_batch(
category_id, current_date, batch_end, guests_adults, guests_children
)
rates.extend(batch_rates)
# Move to next batch
current_date = batch_end + timedelta(days=1)
except Exception as e:
logger.error(f"Failed to fetch rates for category {category_id} starting {current_date}: {e}")
# Skip this batch and continue
current_date = current_date + timedelta(days=7)
# Rate limiting - ALWAYS wait 1.5s between requests, even after errors
await asyncio.sleep(1.5)
return rates
async def get_single_night_rates(
self,
category_id: str,
from_date: date,
to_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> List[Dict]:
"""
Fetch rates with single-night queries for accurate per-day tariff availability.
Unlike get_category_rates which batches, this queries each date individually
as a 1-night stay. This gives accurate tariff_success per night, catching
issues like Valentine's Day blocking only that night, not a whole week.
Much slower but necessary for accurate bookability data.
Args:
category_id: Newbook category ID
from_date: Start date for rates
to_date: End date for rates (inclusive)
guests_adults: Number of adult guests (default 2)
guests_children: Number of child guests (default 0)
Returns:
List of dicts with {date, gross_rate, net_rate, tariffs_data}
"""
rates = []
current_date = from_date
while current_date <= to_date:
try:
# Single-night query for accurate tariff availability
batch_rates = await self._fetch_rates_batch(
category_id, current_date, current_date, guests_adults, guests_children
)
rates.extend(batch_rates)
except Exception as e:
logger.warning(f"Failed to fetch single-night rate for {category_id} on {current_date}: {e}")
# Continue with next date
current_date += timedelta(days=1)
# Rate limiting - wait between each single-night query
await asyncio.sleep(1.0)
return rates
async def fetch_single_date_all_categories(
self,
for_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[str, List[Dict]]:
"""
Fetch single-night rates for ALL categories for one date.
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
"""
return await self._fetch_all_categories_batch(
for_date, guests_adults, guests_children
)
async def fetch_multi_night_for_date(
self,
for_date: date,
nights: int,
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[str, Dict[str, bool]]:
"""
Fetch multi-night availability for ALL categories for one date.
Returns:
Dict of {category_id: {tariff_name: available}}
"""
return await self._fetch_all_categories_multi_night(
for_date, nights, guests_adults, guests_children
)
async def get_all_categories_single_night_rates(
self,
from_date: date,
to_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[str, List[Dict]]:
"""
Fetch rates for ALL categories with single-night queries.
More efficient than get_single_night_rates - omits category_id to get
all categories in a single API call per date. This reduces API calls
from (categories × days) to just (days).
Args:
from_date: Start date for rates
to_date: End date for rates (inclusive)
guests_adults: Number of adult guests (default 2)
guests_children: Number of child guests (default 0)
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}, ...]}
"""
all_rates: Dict[str, List[Dict]] = {}
current_date = from_date
total_days = (to_date - from_date).days + 1
day_count = 0
while current_date <= to_date:
day_count += 1
try:
# Single-night query WITHOUT category_id - returns ALL categories
category_rates = await self._fetch_all_categories_batch(
current_date, guests_adults, guests_children
)
# Merge into all_rates dict
for cat_id, rates in category_rates.items():
if cat_id not in all_rates:
all_rates[cat_id] = []
all_rates[cat_id].extend(rates)
logger.info(f"Fetched {current_date} ({day_count}/{total_days}) - {len(category_rates)} categories")
except Exception as e:
logger.warning(f"Failed to fetch rates for {current_date}: {e}")
# Continue with next date
current_date += timedelta(days=1)
# Rate limiting - wait between each query
await asyncio.sleep(1.0)
return all_rates
async def _fetch_all_categories_batch(
self,
for_date: date,
guests_adults: int,
guests_children: int,
retry_count: int = 0
) -> Dict[str, List[Dict]]:
"""
Fetch rates for ALL categories for a single date.
Omits category_id from request - Newbook returns all available categories.
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
"""
# Single-night query
period_from = f"{for_date.isoformat()} 14:00:00"
period_to = f"{(for_date + timedelta(days=1)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"daily_mode": "true"
# NO category_id - returns all categories
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting with exponential backoff
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1)
logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3")
await asyncio.sleep(wait_time)
return await self._fetch_all_categories_batch(
for_date, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
raise NewbookRatesError(f"API returned failure: {data.get('message')}")
# Parse all categories from response
return self._parse_all_categories_tariffs(data, for_date)
async def _fetch_all_categories_multi_night(
self,
for_date: date,
nights: int,
guests_adults: int = 2,
guests_children: int = 0,
retry_count: int = 0
) -> Dict[str, Dict[str, bool]]:
"""
Fetch multi-night availability for ALL categories for a specific date.
Used to verify that rates with min_stay requirements are actually bookable.
Args:
for_date: Check-in date
nights: Number of nights to query (e.g., 2 for min_stay=2)
guests_adults: Number of adult guests
guests_children: Number of child guests
Returns:
Dict of {category_id: {tariff_name: available}}
"""
# Multi-night query
period_from = f"{for_date.isoformat()} 14:00:00"
period_to = f"{(for_date + timedelta(days=nights)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"daily_mode": "true"
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1)
logger.warning(f"Rate limited (multi-night), waiting {wait_time}s")
await asyncio.sleep(wait_time)
return await self._fetch_all_categories_multi_night(
for_date, nights, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
raise NewbookRatesError(f"API returned failure: {data.get('message')}")
# Parse availability by tariff name for each category
results: Dict[str, Dict[str, bool]] = {}
if not isinstance(data.get("data"), dict):
return results
for key, cat_data in data["data"].items():
if not (key.isdigit() or str(key).isnumeric()):
continue
if not isinstance(cat_data, dict):
continue
category_id = str(key)
tariffs_available = cat_data.get("tariffs_available", [])
results[category_id] = {}
for tariff in tariffs_available:
tariff_name = tariff.get("tariff_name", "")
tariff_label = tariff.get("tariff_label", "")
# Check tariff_success (API returns string "true"/"false")
tariff_success = str(tariff.get("tariff_success", False)).lower() in ("true", "1")
# Available if API says success, OR if rates are quoted and no restriction message
is_available = tariff_success or (
bool(tariff.get("tariffs_quoted")) and not tariff.get("tariff_message")
)
# Store under both tariff_name and tariff_label for flexible matching
results[category_id][tariff_name] = is_available
if tariff_label and tariff_label != tariff_name:
results[category_id][tariff_label] = is_available
return results
async def get_multi_night_availability(
self,
dates_by_nights: Dict[int, List[date]],
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[date, Dict[str, Dict[str, bool]]]:
"""
Fetch multi-night availability for specific dates grouped by stay length.
Checks if a tariff is available when booking N nights starting from each date.
Args:
dates_by_nights: Dict of {nights: [dates]} e.g., {2: [date1, date2], 3: [date3]}
guests_adults: Number of adult guests
guests_children: Number of child guests
Returns:
Dict of {date: {category_id: {tariff_name: available}}}
"""
results: Dict[date, Dict[str, Dict[str, bool]]] = {}
total_queries = sum(len(dates) for dates in dates_by_nights.values())
query_count = 0
for nights, dates in dates_by_nights.items():
for query_date in dates:
query_count += 1
try:
result = await self._fetch_all_categories_multi_night(
query_date, nights, guests_adults, guests_children
)
results[query_date] = result
logger.info(f"Multi-night check {query_count}/{total_queries}: {query_date} ({nights} nights)")
except Exception as e:
logger.warning(f"Failed multi-night check for {query_date}: {e}")
# Rate limiting
await asyncio.sleep(1.0)
return results
async def _fetch_rates_batch(
self,
category_id: str,
from_date: date,
to_date: date,
guests_adults: int,
guests_children: int,
retry_count: int = 0
) -> List[Dict]:
"""
Fetch rates for a batch of dates (up to 7 days).
Handles minimum stay restrictions by extending the period and
extracting only the dates we need.
Returns:
List of dicts with {date, gross_rate, net_rate}
"""
# Format dates with times (check-in 14:00, check-out 10:00)
period_from = f"{from_date.isoformat()} 14:00:00"
period_to = f"{(to_date + timedelta(days=1)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"category_id": category_id,
"daily_mode": "true" # Get per-night breakdown
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting with exponential backoff
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1) # 60s, 120s, 180s
logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3")
await asyncio.sleep(wait_time)
return await self._fetch_rates_batch(
category_id, from_date, to_date, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
# Check if minimum stay restriction
categories = data.get("data", {}).get("categories", [])
if categories:
cat = categories[0] if isinstance(categories, list) else categories.get(category_id, {})
min_periods = cat.get("minimum_periods", 1)
if min_periods > 1:
# Extend the stay to meet minimum and retry
extended_to = from_date + timedelta(days=min_periods)
logger.info(f"Minimum stay {min_periods} nights for category {category_id}, extending to {extended_to}")
return await self._fetch_rates_with_min_stay(
category_id, from_date, to_date, extended_to,
guests_adults, guests_children
)
raise NewbookRatesError(f"API returned failure: {data.get('message')}")
# Parse tariffs_quoted from response
return self._parse_tariffs(data, from_date, to_date)
async def _fetch_rates_with_min_stay(
self,
category_id: str,
from_date: date,
to_date: date,
extended_to: date,
guests_adults: int,
guests_children: int,
retry_count: int = 0
) -> List[Dict]:
"""
Fetch rates with extended period for minimum stay requirement.
Args:
category_id: Newbook category ID
from_date: Original start date
to_date: Original end date (dates we want)
extended_to: Extended end date to meet minimum stay
guests_adults: Number of adults
guests_children: Number of children
Returns:
List of rates for the original date range only
"""
period_from = f"{from_date.isoformat()} 14:00:00"
period_to = f"{(extended_to + timedelta(days=1)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"category_id": category_id,
"daily_mode": "true"
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting with exponential backoff
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1)
logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry")
await asyncio.sleep(wait_time)
return await self._fetch_rates_with_min_stay(
category_id, from_date, to_date, extended_to, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
raise NewbookRatesError(f"API returned failure even with extended stay: {data.get('message')}")
# Parse tariffs but only return dates in our original range
return self._parse_tariffs(data, from_date, to_date)
def _parse_tariffs(self, data: dict, from_date: date, to_date: date) -> List[Dict]:
"""
Parse tariffs from API response.
With daily_mode=true, the API returns tariffs_quoted as a dict keyed by date.
Falls back to tariffs_available average if tariffs_quoted not available.
Args:
data: Full API response
from_date: Start date to include
to_date: End date to include
Returns:
List of dicts with {date, gross_rate, net_rate, tariffs_data}
tariffs_data contains all available tariff options for rate report
"""
rates = []
tariffs_quoted = {}
fallback_rate = None
inventory_items = []
all_tariffs_available = [] # Store all tariff options for reporting
# Find tariffs data in the response
if isinstance(data.get("data"), dict):
for key in data["data"].keys():
# Category IDs are numeric strings
if key.isdigit() or key.isnumeric():
cat_data = data["data"][key]
if isinstance(cat_data, dict):
tariffs_available = cat_data.get("tariffs_available", [])
all_tariffs_available = tariffs_available # Capture all options
if tariffs_available:
first_tariff = tariffs_available[0]
# tariffs_quoted is a dict keyed by date string
tariffs_quoted = first_tariff.get("tariffs_quoted", {})
# inventory_items are at tariff level (total for whole stay)
inventory_items = first_tariff.get("inventory_items", [])
# Fallback average rate
fallback_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0))
break
# If we have per-night tariffs_quoted dict, parse it
if isinstance(tariffs_quoted, dict) and tariffs_quoted:
num_nights = len(tariffs_quoted)
# Calculate per-night inventory item amount for items already included in tariff
included_inventory_per_night = Decimal('0')
for item in inventory_items:
already_included = item.get('amount_already_included_in_tariff_total', '')
if str(already_included).lower() == 'true':
total_amount = Decimal(str(item.get('amount', 0) or 0))
included_inventory_per_night += total_amount / num_nights
for date_str, tariff in tariffs_quoted.items():
try:
stay_date = date.fromisoformat(date_str)
except ValueError:
continue
# Only include dates in our range
if stay_date < from_date or stay_date > to_date:
continue
gross_rate = Decimal(str(tariff.get('amount', 0) or 0))
# Net = (gross - included_inventory_per_night) / (1 + VAT)
gross_after_inventory = gross_rate - included_inventory_per_night
net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01'))
# Build tariffs_data with day-specific rates
tariffs_data = self._build_tariffs_summary(all_tariffs_available, stay_date)
rates.append({
'date': stay_date,
'gross_rate': float(gross_rate),
'net_rate': float(net_rate),
'tariffs_data': tariffs_data
})
return rates
# Fallback: use average_nightly_tariff and apply to all dates
if fallback_rate and fallback_rate > 0:
net_rate = (fallback_rate / (1 + self.vat_rate)).quantize(Decimal('0.01'))
current_date = from_date
while current_date <= to_date:
# Build tariffs_data (no day-specific rates in fallback)
tariffs_data = self._build_tariffs_summary(all_tariffs_available, current_date)
rates.append({
'date': current_date,
'gross_rate': float(fallback_rate),
'net_rate': float(net_rate),
'tariffs_data': tariffs_data
})
current_date += timedelta(days=1)
return rates
logger.warning(f"No rate found in response for {from_date} to {to_date}")
return rates
def _build_tariffs_summary(self, tariffs_available: list, for_date: date = None) -> dict:
"""
Build a summary of all available tariff options for rate reporting.
Args:
tariffs_available: List of tariff dicts from API response
for_date: Optional specific date to extract day-specific rates
Returns:
Dict with tariff summaries - tariff_count and list of tariff details
"""
if not tariffs_available:
return {}
summary = {
'tariff_count': len(tariffs_available),
'tariffs': []
}
date_key = for_date.isoformat() if for_date else None
for idx, tariff in enumerate(tariffs_available):
# Get day-specific rate from tariffs_quoted if available
day_rate = None
if date_key:
tariffs_quoted = tariff.get('tariffs_quoted', {})
if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted:
day_quote = tariffs_quoted[date_key]
if isinstance(day_quote, dict):
day_rate = float(day_quote.get('amount', 0) or 0)
else:
day_rate = float(day_quote or 0)
# API uses tariff_label for the name
message = tariff.get('tariff_message', '')
# Extract minimum stay from message or dedicated field
min_stay = tariff.get('minimum_nights', None)
if min_stay is None and message:
# Try to parse from message like "Minimum 2 nights" or "2 Night Minimum"
import re
match = re.search(r'(\d+)\s*[Nn]ight\s*[Mm]inimum', message)
if not match:
match = re.search(r'[Mm]inimum\s+(\d+)\s*(?:night|period)', message)
if match:
min_stay = int(match.group(1))
# Extract advance booking requirement from message
min_advance_days = None
if message:
import re
advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
if advance_match:
min_advance_days = int(advance_match.group(1))
tariff_info = {
'name': tariff.get('tariff_label', 'Unknown'),
'description': tariff.get('tariff_short_description', ''),
'rate': day_rate, # Day-specific rate (None if not available)
'average_nightly': float(tariff.get('average_nightly_tariff', 0) or 0),
'success': str(tariff.get('tariff_success', False)).lower() in ('true', '1'),
'message': message,
'sort_order': idx, # Preserve Newbook ordering
'min_stay': min_stay, # Minimum nights required (if any)
'min_advance_days': min_advance_days, # Advance booking requirement (if any)
}
summary['tariffs'].append(tariff_info)
return summary
def _parse_all_categories_tariffs(self, data: dict, for_date: date) -> Dict[str, List[Dict]]:
"""
Parse tariffs from API response for ALL categories.
When category_id is omitted, data.data contains category IDs as keys,
each with their own tariffs_available.
Args:
data: Full API response
for_date: The date we queried
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
"""
results: Dict[str, List[Dict]] = {}
if not isinstance(data.get("data"), dict):
return results
for key, cat_data in data["data"].items():
# Category IDs are numeric strings like "1", "8", etc.
if not (key.isdigit() or str(key).isnumeric()):
continue
if not isinstance(cat_data, dict):
continue
category_id = str(key)
tariffs_available = cat_data.get("tariffs_available", [])
if not tariffs_available:
continue
# Get the first (best) tariff for gross/net calculation
first_tariff = tariffs_available[0]
tariffs_quoted = first_tariff.get("tariffs_quoted", {})
inventory_items = first_tariff.get("inventory_items", [])
# Get rate for this date
date_key = for_date.isoformat()
gross_rate = Decimal('0')
net_rate = Decimal('0')
if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted:
day_tariff = tariffs_quoted[date_key]
gross_rate = Decimal(str(day_tariff.get('amount', 0) or 0))
# Calculate included inventory per night
included_inventory = Decimal('0')
for item in inventory_items:
already_included = item.get('amount_already_included_in_tariff_total', '')
if str(already_included).lower() == 'true':
included_inventory += Decimal(str(item.get('amount', 0) or 0))
gross_after_inventory = gross_rate - included_inventory
net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01'))
else:
# Fallback to average
gross_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0))
net_rate = (gross_rate / (1 + self.vat_rate)).quantize(Decimal('0.01'))
# Build tariffs summary for all options
tariffs_data = self._build_tariffs_summary(tariffs_available, for_date)
results[category_id] = [{
'date': for_date,
'gross_rate': float(gross_rate),
'net_rate': float(net_rate),
'tariffs_data': tariffs_data
}]
return results

20
backend/services/scraper_backends/__init__.py Normal file
View file

@ -0,0 +1,20 @@
"""
Scraper backends for booking.com rate scraping.
Provides pluggable backends to allow switching between:
- playwright_local: Direct Playwright (default)
- playwright_proxy: Playwright with rotating proxies (future)
- apify_backend: Apify scraping service (future)
"""
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
from .playwright_local import PlaywrightLocalBackend
__all__ = [
'ScraperBackend',
'ScraperResult',
'HotelData',
'RateData',
'AvailabilityStatus',
'PlaywrightLocalBackend',
]

152
backend/services/scraper_backends/base.py Normal file
View file

@ -0,0 +1,152 @@
"""
Abstract base class for booking.com scraper backends.
Defines the interface that all scraper backends must implement,
allowing easy switching between local Playwright, proxied Playwright,
or external services like Apify.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import date
from decimal import Decimal
from typing import List, Optional, Dict, Any
from enum import Enum
class AvailabilityStatus(str, Enum):
"""Availability status for a hotel rate."""
AVAILABLE = 'available' # Rate found, bookable
SOLD_OUT = 'sold_out' # Hotel shows no availability
NO_DATA = 'no_data' # Couldn't determine (scraper issue)
@dataclass
class RateData:
"""Rate data for a single hotel on a single date."""
hotel_id: Optional[str] = None # Our internal hotel_id (filled after DB lookup)
booking_com_id: str = '' # Hotel ID from booking.com
rate_date: date = None
availability_status: AvailabilityStatus = AvailabilityStatus.NO_DATA
rate_gross: Optional[Decimal] = None
currency: str = 'GBP'
room_type: Optional[str] = None
breakfast_included: Optional[bool] = None
free_cancellation: Optional[bool] = None
no_prepayment: Optional[bool] = None
rooms_left: Optional[int] = None # "Only X rooms left"
available_qty: Optional[int] = None # Future: from hotel page dropdown
@dataclass
class HotelData:
"""Hotel data discovered from search results."""
booking_com_id: str
name: str
booking_com_url: Optional[str] = None
star_rating: Optional[Decimal] = None
review_score: Optional[Decimal] = None
review_count: Optional[int] = None
@dataclass
class ScraperResult:
"""Result from a scraping operation."""
success: bool
blocked: bool = False # True if anti-scrape blocking detected
block_reason: Optional[str] = None # CAPTCHA, rate limit, etc.
hotels: List[HotelData] = field(default_factory=list)
rates: List[RateData] = field(default_factory=list)
error_message: Optional[str] = None
page_content_sample: Optional[str] = None # For debugging
class ScraperBackend(ABC):
"""
Abstract base class for scraper backends.
All backends must implement these methods to provide a consistent
interface for the main booking_scraper.py service.
"""
# Common block detection signals
BLOCK_SIGNALS = [
'captcha',
'unusual traffic',
'access denied',
'please verify',
'too many requests',
'are you a robot',
'verify you are human',
'security check',
]
@abstractmethod
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name (e.g., "Bowness-on-Windermere")
check_in: Check-in date
check_out: Check-out date (typically check_in + 1 for single night)
adults: Number of adults for search
pages: Number of search result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
pass
@abstractmethod
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape an individual hotel page for detailed rates.
Future expansion - not used in initial implementation.
Will provide available_qty from room dropdowns.
Args:
hotel_url: Full booking.com URL for the hotel
check_in: Check-in date
check_out: Check-out date
adults: Number of adults
Returns:
ScraperResult with detailed rate information
"""
pass
@abstractmethod
async def close(self):
"""Clean up any resources (browser instances, etc.)."""
pass
def detect_blocking(self, page_content: str) -> tuple[bool, Optional[str]]:
"""
Check if page content shows anti-scrape response.
Args:
page_content: HTML content of the page
Returns:
Tuple of (is_blocked, reason)
"""
content_lower = page_content.lower()
for signal in self.BLOCK_SIGNALS:
if signal in content_lower:
return True, signal
return False, None

401
backend/services/scraper_backends/playwright_local.py Normal file
View file

@ -0,0 +1,401 @@
"""
Local Playwright backend for booking.com scraping.
Uses Playwright with Chromium to scrape search results.
No proxy - direct connection. Suitable for low-volume scraping.
"""
import asyncio
import logging
import random
import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import (
ScraperBackend,
ScraperResult,
HotelData,
RateData,
AvailabilityStatus
)
logger = logging.getLogger(__name__)
class PlaywrightLocalBackend(ScraperBackend):
"""
Local Playwright backend using Chromium.
Features:
- Rotates user agents
- Random delays between requests
- Mimics human scroll behavior
- Uses data-testid selectors for stability
"""
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
]
def __init__(self, proxy_config: dict = None):
"""
Initialize the backend.
Args:
proxy_config: Optional proxy configuration (for future use)
"""
self.proxy_config = proxy_config
self._playwright = None
self._browser: Optional[Browser] = None
async def _ensure_browser(self) -> Browser:
"""Ensure browser is running, start if needed."""
if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--no-sandbox',
'--disable-dev-shm-usage',
]
)
return self._browser
async def _create_context(self) -> BrowserContext:
"""Create a new browser context with random user agent."""
browser = await self._ensure_browser()
context = await browser.new_context(
user_agent=random.choice(self.USER_AGENTS),
viewport={'width': 1920, 'height': 1080},
locale='en-GB',
timezone_id='Europe/London',
)
return context
def _build_search_url(
self,
location: str,
check_in: date,
check_out: date,
adults: int,
offset: int = 0
) -> str:
"""Build booking.com search URL with parameters."""
params = {
'ss': location,
'checkin': check_in.isoformat(),
'checkout': check_out.isoformat(),
'group_adults': adults,
'no_rooms': 1,
'group_children': 0,
}
if offset > 0:
params['offset'] = offset
return f"https://www.booking.com/searchresults.en-gb.html?{urlencode(params)}"
def _parse_price(self, price_text: str) -> Optional[Decimal]:
"""Parse price from text like '£150' or 'GBP 150'."""
if not price_text:
return None
# Remove currency symbols and extract number
cleaned = re.sub(r'[£$€,\s]', '', price_text)
# Find first number (including decimals)
match = re.search(r'[\d,]+(?:\.\d{2})?', cleaned)
if match:
try:
return Decimal(match.group().replace(',', ''))
except InvalidOperation:
return None
return None
def _extract_hotel_id(self, url: str) -> Optional[str]:
"""Extract hotel ID from booking.com URL."""
if not url:
return None
# URL format: /hotel/gb/hotel-name.en-gb.html or ?dest_id=123
# Try to extract from URL path
match = re.search(r'/hotel/[a-z]{2}/([^/]+)\.', url)
if match:
return match.group(1)
# Try dest_id parameter
match = re.search(r'dest_id=(-?\d+)', url)
if match:
return match.group(1)
return None
async def _human_like_scroll(self, page: Page):
"""Simulate human-like scrolling behavior."""
# Scroll down in increments
for _ in range(3):
await page.mouse.wheel(0, random.randint(300, 600))
await asyncio.sleep(random.uniform(0.3, 0.8))
async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
"""Extract hotel and rate data from search results page."""
hotels = []
rates = []
# Wait for property cards - booking.com uses data-testid
try:
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
except Exception as e:
logger.warning(f"No property cards found: {e}")
return hotels, rates
# Get all property cards
cards = await page.query_selector_all('[data-testid="property-card"]')
logger.info(f"Found {len(cards)} property cards")
for card in cards:
try:
hotel = HotelData(booking_com_id='', name='')
rate = RateData(rate_date=rate_date)
# Hotel name
name_el = await card.query_selector('[data-testid="title"]')
if name_el:
hotel.name = (await name_el.inner_text()).strip()
if not hotel.name:
continue # Skip if no name found
# Hotel URL and ID
link_el = await card.query_selector('[data-testid="title-link"]')
if link_el:
hotel.booking_com_url = await link_el.get_attribute('href')
hotel.booking_com_id = self._extract_hotel_id(hotel.booking_com_url) or ''
rate.booking_com_id = hotel.booking_com_id
# Star rating - look for star icons or rating text
stars_el = await card.query_selector('[data-testid="rating-stars"]')
if stars_el:
stars_text = await stars_el.get_attribute('aria-label') or ''
match = re.search(r'(\d+)', stars_text)
if match:
hotel.star_rating = Decimal(match.group(1))
# Review score
score_el = await card.query_selector('[data-testid="review-score"]')
if score_el:
score_text = await score_el.inner_text()
match = re.search(r'([\d.]+)', score_text)
if match:
try:
hotel.review_score = Decimal(match.group(1))
except InvalidOperation:
pass
# Check for no availability message FIRST
no_avail_el = await card.query_selector('[data-testid="availability-message"]')
if no_avail_el:
avail_text = (await no_avail_el.inner_text()).lower()
if 'no availability' in avail_text or 'sold out' in avail_text:
rate.availability_status = AvailabilityStatus.SOLD_OUT
hotels.append(hotel)
rates.append(rate)
continue
# Price
price_el = await card.query_selector('[data-testid="price-and-discounted-price"]')
if not price_el:
# Try alternative selector
price_el = await card.query_selector('[data-testid="price"]')
if price_el:
price_text = await price_el.inner_text()
rate.rate_gross = self._parse_price(price_text)
if rate.rate_gross:
rate.availability_status = AvailabilityStatus.AVAILABLE
# Room type
room_el = await card.query_selector('[data-testid="recommended-units"]')
if room_el:
rate.room_type = (await room_el.inner_text()).strip()
# Rate option badges - try multiple selectors
# Breakfast included
breakfast_el = await card.query_selector('[data-testid="breakfast-included"]')
if not breakfast_el:
# Check text content for breakfast mentions
card_text = (await card.inner_text()).lower()
rate.breakfast_included = 'breakfast included' in card_text
else:
rate.breakfast_included = True
# Free cancellation
cancel_el = await card.query_selector('[data-testid="cancellation-policy"]')
if cancel_el:
cancel_text = (await cancel_el.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in cancel_text
else:
card_text = (await card.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in card_text
# No prepayment
prepay_el = await card.query_selector('[data-testid="no-prepayment"]')
if prepay_el:
rate.no_prepayment = True
else:
card_text = (await card.inner_text()).lower()
rate.no_prepayment = 'no prepayment' in card_text
# Rooms left / scarcity indicator
scarcity_el = await card.query_selector('[data-testid="availability-rate"]')
if scarcity_el:
scarcity_text = await scarcity_el.inner_text()
match = re.search(r'(\d+)\s*room', scarcity_text.lower())
if match:
rate.rooms_left = int(match.group(1))
hotels.append(hotel)
rates.append(rate)
except Exception as e:
logger.warning(f"Error extracting hotel data: {e}")
continue
return hotels, rates
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name
check_in: Check-in date
check_out: Check-out date (check_in + 1 for single night rate)
adults: Number of adults
pages: Number of result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
all_hotels = []
all_rates = []
seen_hotel_ids = set()
context = None
page = None
try:
context = await self._create_context()
page = await context.new_page()
for page_num in range(pages):
# Random delay between pages (3-7 seconds)
if page_num > 0:
delay = random.uniform(3, 7)
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
await asyncio.sleep(delay)
# Build URL with offset for pagination (25 results per page)
url = self._build_search_url(
location, check_in, check_out, adults,
offset=page_num * 25
)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='networkidle', timeout=30000)
except Exception as e:
logger.warning(f"Page load timeout, continuing: {e}")
# Check for blocking
content = await page.content()
is_blocked, reason = self.detect_blocking(content)
if is_blocked:
logger.warning(f"Blocking detected: {reason}")
return ScraperResult(
success=False,
blocked=True,
block_reason=reason,
hotels=all_hotels,
rates=all_rates,
page_content_sample=content[:1000]
)
# Human-like scrolling
await self._human_like_scroll(page)
# Extract data
hotels, rates = await self._extract_search_results(page, check_in)
# Deduplicate by booking_com_id
for hotel, rate in zip(hotels, rates):
if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids:
seen_hotel_ids.add(hotel.booking_com_id)
all_hotels.append(hotel)
all_rates.append(rate)
logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique")
return ScraperResult(
success=True,
blocked=False,
hotels=all_hotels,
rates=all_rates
)
except Exception as e:
logger.error(f"Scrape error: {e}")
return ScraperResult(
success=False,
blocked=False,
error_message=str(e),
hotels=all_hotels,
rates=all_rates
)
finally:
if page:
await page.close()
if context:
await context.close()
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape individual hotel page for detailed rates.
Future expansion - placeholder for now.
Will extract available_qty from room dropdowns.
"""
# Not implemented in Phase 2a
logger.warning("scrape_hotel_page not yet implemented")
return ScraperResult(
success=False,
error_message="Hotel page scraping not yet implemented"
)
async def close(self):
"""Clean up browser resources."""
if self._browser:
await self._browser.close()
self._browser = None
if self._playwright:
await self._playwright.stop()
self._playwright = None