diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 075c543..fd490ab 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -675,7 +675,8 @@ async def get_competitor_matrix( WHERE {tier_filter} AND h.is_active = TRUE AND r.rate_date >= :from_date AND r.rate_date <= :to_date - ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC + AND (r.max_persons IS NULL OR r.max_persons = 2) + ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC, r.rate_gross ASC NULLS LAST """), {'from_date': start, 'to_date': end} ) @@ -714,7 +715,8 @@ async def get_competitor_matrix( AND r.rate_date >= :from_date AND r.rate_date <= :to_date AND r.availability_status = 'available' AND r.rate_gross IS NOT NULL - ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC + AND (r.max_persons IS NULL OR r.max_persons = 2) + ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC, r.rate_gross ASC NULLS LAST """), {'from_date': start, 'to_date': end} ) diff --git a/backend/schema.sql b/backend/schema.sql index b85c902..f23570e 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -17,7 +17,7 @@ INSERT INTO system_config (config_key, config_value, description) VALUES ('booking_scraper_enabled', 'false', 'Enable automatic booking.com rate scraping (true/false)'), ('booking_scraper_paused', 'false', 'Scraper temporarily paused due to blocking (true/false)'), ('booking_scraper_pause_until', NULL, 'ISO datetime when pause expires'), -('booking_scraper_backend', 'playwright_local', 'Scraper backend: playwright_local | playwright_proxy | apify'), +('booking_scraper_backend', 'playwright_local', 'Scraper backend: playwright_local | playwright_hotel_page | apify'), ('booking_scraper_daily_time', '05:30', 'Daily scrape time (HH:MM)'), ('booking_scraper_proxy_url', NULL, 'Proxy URL for playwright_proxy backend'), ('booking_scraper_proxy_username', NULL, 'Proxy username for playwright_proxy backend'), @@ -128,6 +128,10 @@ CREATE TABLE IF NOT EXISTS booking_com_rates ( ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT; ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255); +-- Hotel page scraper: rate plan identifier and occupancy per row +ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS rate_plan_id TEXT; +ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS max_persons INTEGER; + CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC); diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 3a070a0..fa8b0c1 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -25,7 +25,10 @@ from sqlalchemy.orm import Session from database import SyncSessionLocal from services import proxy as proxy_util -from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus +from .scraper_backends import ( + ScraperBackend, PlaywrightLocalBackend, PlaywrightHotelPageBackend, + HotelData, RateData, AvailabilityStatus, +) logger = logging.getLogger(__name__) @@ -94,12 +97,17 @@ def get_scraper_backend(db: Session) -> ScraperBackend: # Future: Apify backend raise NotImplementedError("Apify backend not yet implemented") + proxy_cfg = proxy_util.load_config(db) + + if backend_type == 'playwright_hotel_page': + return PlaywrightHotelPageBackend(proxy_config=proxy_cfg) + if backend_type not in ('playwright_local', 'playwright_proxy'): logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local") # Proxy config resolved by the shared module: system_config is authoritative # when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct). - return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db)) + return PlaywrightLocalBackend(proxy_config=proxy_cfg) def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: @@ -189,9 +197,11 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID): text(""" INSERT INTO booking_com_rates (hotel_id, rate_date, availability_status, rate_gross, currency, room_type, - breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id) + breakfast_included, free_cancellation, no_prepayment, rooms_left, + rate_plan_id, max_persons, scrape_batch_id) VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type, - :breakfast, :cancel, :prepay, :rooms_left, :batch_id) + :breakfast, :cancel, :prepay, :rooms_left, + :rate_plan_id, :max_persons, :batch_id) """), { 'hotel_id': hotel_id, @@ -204,6 +214,8 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID): 'cancel': rate.free_cancellation, 'prepay': rate.no_prepayment, 'rooms_left': rate.rooms_left, + 'rate_plan_id': rate.rate_plan_id, + 'max_persons': rate.max_persons, 'batch_id': str(batch_id), } ) @@ -282,6 +294,69 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60): return len(cleaned) +def get_active_hotels(db: Session) -> List[Dict[str, Any]]: + """Return all active hotels that have a booking_com_url (needed for hotel-page scraping).""" + rows = db.execute( + text(""" + SELECT id, booking_com_id, name, booking_com_url + FROM booking_com_hotels + WHERE is_active = TRUE AND booking_com_url IS NOT NULL + ORDER BY display_order, id + """) + ).fetchall() + return [dict(r._mapping) for r in rows] + + +async def scrape_hotel_date( + db: Session, + hotel: Dict[str, Any], + rate_date: date, + backend: 'PlaywrightHotelPageBackend', + batch_id: uuid.UUID, + adults: int = 2, + max_retries: int = 2, +) -> Dict[str, Any]: + """ + Scrape all rate plans for one hotel on one date via the hotel page backend. + + On block detection, the backend rotates its proxy context automatically. + We retry up to max_retries times to handle the rotation. + """ + check_out = rate_date + timedelta(days=1) + hotel_url = hotel['booking_com_url'] + + for attempt in range(max_retries + 1): + result = await backend.scrape_hotel_page(hotel_url, rate_date, check_out, adults) + + if result.success: + rates_saved = 0 + for rate in result.rates: + rate.rate_date = rate_date # ensure date is set + try: + save_rate(db, rate, hotel['id'], batch_id) + rates_saved += 1 + except Exception as e: + logger.warning(f"Error saving rate plan for {hotel['name']} {rate_date}: {e}") + db.rollback() + db.commit() + return {'success': True, 'blocked': False, 'rates_count': rates_saved} + + if result.blocked and attempt < max_retries: + logger.info( + f"Hotel {hotel['name']} {rate_date} blocked (attempt {attempt + 1}), retrying…" + ) + continue + + return { + 'success': False, + 'blocked': result.blocked, + 'error': result.error_message, + 'rates_count': 0, + } + + return {'success': False, 'blocked': True, 'rates_count': 0} + + async def scrape_date( db: Session, rate_date: date, @@ -438,6 +513,13 @@ def get_scraper_concurrency(db: Session) -> int: return 6 +def _is_hotel_page_mode(db: Session) -> bool: + row = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'") + ).fetchone() + return (row and row.config_value) == 'playwright_hotel_page' + + def _effective_concurrency(db: Session, n_jobs: int) -> int: """Clamp configured concurrency to the workload, and force serial when the proxy is off — N workers would share one IP and hammer it, worse than 1.""" @@ -451,6 +533,82 @@ def _effective_concurrency(db: Session, n_jobs: int) -> int: return max(1, min(configured, n_jobs)) +async def _scrape_hotels_concurrent( + hotel_date_jobs: List[Tuple[Dict[str, Any], date]], + concurrency: int, + batch_id: uuid.UUID, + adults: int = 2, +) -> Dict[str, int]: + """ + Scrape (hotel, date) pairs using the hotel-page backend. + + Jobs are sharded by HOTEL (not date) so each worker keeps its proxy session + alive across all dates for one hotel — looks like a single user checking + availability for a trip. + """ + # Group by hotel to get stable shards + hotels_seen: List[Dict[str, Any]] = [] + hotel_dates: Dict[int, List[date]] = {} + for hotel, rate_date in hotel_date_jobs: + hid = hotel['id'] + if hid not in hotel_dates: + hotel_dates[hid] = [] + hotels_seen.append(hotel) + hotel_dates[hid].append(rate_date) + + # Shard hotels across workers + shards: List[List[Dict[str, Any]]] = [hotels_seen[i::concurrency] for i in range(concurrency)] + shards = [s for s in shards if s] + + async def worker(hotel_shard: List[Dict[str, Any]], widx: int) -> Dict[str, int]: + acc = {'rates': 0, 'completed': 0, 'failed': 0, 'blocked': 0} + wdb = SyncSessionLocal() + backend = PlaywrightHotelPageBackend(proxy_config=proxy_util.load_config(wdb)) + try: + for hotel in hotel_shard: + for rate_date in hotel_dates[hotel['id']]: + try: + result = await scrape_hotel_date(wdb, hotel, rate_date, backend, batch_id, adults) + except Exception as e: + logger.error(f"[worker {widx}] {hotel['name']} {rate_date} crashed: {e}") + try: + wdb.rollback() + except Exception: + pass + acc['failed'] += 1 + continue + + if result.get('blocked'): + acc['blocked'] += 1 + acc['failed'] += 1 + elif result['success']: + acc['rates'] += result['rates_count'] + acc['completed'] += 1 + else: + acc['failed'] += 1 + finally: + try: + await backend.close() + except Exception: + pass + wdb.close() + return acc + + results = await asyncio.gather( + *(worker(shard, i) for i, shard in enumerate(shards)), + return_exceptions=True, + ) + + agg = {'rates': 0, 'completed': 0, 'failed': 0, 'blocked': 0} + for r in results: + if isinstance(r, Exception): + logger.error(f"Hotel-page scrape worker crashed: {r}") + continue + for k in agg: + agg[k] += r[k] + return agg + + def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None): """mark_queue_item that never raises — a marking failure shouldn't kill a worker.""" if queue_id is None: @@ -565,12 +723,105 @@ async def run_manual_scrape( _release_scrape_lock() +async def _run_hotel_page_scrape( + db: Session, + from_date: date, + to_date: date, + scrape_type: str, +) -> Dict[str, Any]: + """ + Hotel-page scrape mode: load each known hotel's property page per date. + + Unlike search-results mode, this scrapes known hotels (from booking_com_hotels + WHERE is_active AND booking_com_url IS NOT NULL) rather than discovering them + from search results. Workers are sharded by hotel so each worker's proxy session + covers all dates for one hotel before moving to the next. + """ + hotels = get_active_hotels(db) + if not hotels: + return { + 'success': False, + 'error': 'No active hotels with booking_com_url in database. ' + 'Run a search-results scrape first to populate hotels.', + } + + dates: List[date] = [] + current_date = from_date + while current_date <= to_date: + dates.append(current_date) + current_date += timedelta(days=1) + + # Build (hotel, date) job pairs + hotel_date_jobs: List[Tuple[Dict[str, Any], date]] = [ + (hotel, d) for hotel in hotels for d in dates + ] + + batch_id = create_scrape_batch(db, scrape_type) + db.execute( + text("UPDATE booking_scrape_log SET dates_queued = :n WHERE batch_id = :bid"), + {'n': len(hotel_date_jobs), 'bid': str(batch_id)} + ) + db.commit() + + concurrency = _effective_concurrency(db, len(hotels)) + logger.info( + f"Hotel-page scrape {from_date}..{to_date}: " + f"{len(hotels)} hotels × {len(dates)} dates = {len(hotel_date_jobs)} jobs, " + f"{concurrency} worker(s)" + ) + + try: + config = get_scrape_config(db) + adults = config['adults'] if config else 2 + agg = await _scrape_hotels_concurrent(hotel_date_jobs, concurrency, batch_id, adults) + + status = 'completed' if agg['completed'] else 'failed' + update_scrape_batch( + db, batch_id, + status=status, + hotels_found=len(hotels), + rates_scraped=agg['rates'], + ) + db.execute( + text(""" + UPDATE booking_scrape_log SET + dates_completed = :completed, dates_failed = :failed + WHERE batch_id = :bid + """), + {'completed': agg['completed'], 'failed': agg['failed'], 'bid': str(batch_id)} + ) + db.commit() + + return { + 'success': agg['completed'] > 0, + 'blocked': agg['blocked'] > 0, + 'dates_completed': agg['completed'], + 'dates_failed': agg['failed'], + 'hotels_found': len(hotels), + 'rates_scraped': agg['rates'], + } + + except Exception as e: + logger.error(f"Hotel-page scrape error: {e}") + update_scrape_batch(db, batch_id, status='failed', error_message=str(e)) + return { + 'success': False, + 'error': str(e), + 'dates_completed': 0, + 'dates_failed': len(hotel_date_jobs), + } + + async def _run_manual_scrape_locked( db: Session, from_date: date, to_date: date ) -> Dict[str, Any]: - # Get config + # Hotel-page mode: iterate known hotels × dates + if _is_hotel_page_mode(db): + return await _run_hotel_page_scrape(db, from_date, to_date, 'manual') + + # Search-results mode (original) config = get_scrape_config(db) if not config: return { @@ -751,6 +1002,17 @@ async def process_queue(db: Session) -> Dict[str, Any]: async def _process_queue_locked(db: Session) -> Dict[str, Any]: + # Hotel-page mode ignores the queue and just scrapes the queued date range directly + if _is_hotel_page_mode(db): + items = get_pending_queue_items(db, limit=200) + if not items: + return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'} + dates = [it['rate_date'] for it in items] + # Mark all as completed (hotel-page scrape manages its own tracking) + for it in items: + _safe_mark_queue(db, it['id'], 'completed') + return await _run_hotel_page_scrape(db, min(dates), max(dates), 'scheduled') + # Get config config = get_scrape_config(db) if not config: diff --git a/backend/services/scraper_backends/__init__.py b/backend/services/scraper_backends/__init__.py index d54c86a..fbe0d43 100644 --- a/backend/services/scraper_backends/__init__.py +++ b/backend/services/scraper_backends/__init__.py @@ -9,6 +9,7 @@ Provides pluggable backends to allow switching between: from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus from .playwright_local import PlaywrightLocalBackend +from .playwright_hotel_page import PlaywrightHotelPageBackend __all__ = [ 'ScraperBackend', @@ -17,4 +18,5 @@ __all__ = [ 'RateData', 'AvailabilityStatus', 'PlaywrightLocalBackend', + 'PlaywrightHotelPageBackend', ] diff --git a/backend/services/scraper_backends/base.py b/backend/services/scraper_backends/base.py index 92d74f3..34f0b43 100644 --- a/backend/services/scraper_backends/base.py +++ b/backend/services/scraper_backends/base.py @@ -36,6 +36,8 @@ class RateData: no_prepayment: Optional[bool] = None rooms_left: Optional[int] = None # "Only X rooms left" available_qty: Optional[int] = None # Future: from hotel page dropdown + rate_plan_id: Optional[str] = None # block_id from hotel page (identifies rate variant) + max_persons: Optional[int] = None # Occupancy this rate applies to @dataclass diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py new file mode 100644 index 0000000..0c9d8ea --- /dev/null +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -0,0 +1,328 @@ +""" +Booking.com Hotel Page Scraper Backend + +Scrapes individual hotel property pages instead of search results. +Returns all room types and rate plan variants per hotel per date. + +Advantages over search-results approach: +- Individual hotel pages are not Cloudflare-protected (search results are) +- Captures every room type, rate plan, meal plan, and availability count +- Proxy reuse: one residential IP stays valid across many hotel pages + (looks like a human browsing properties), rotates only when blocked + +Rate plan variants per room type (typical): + 2-adult + room only + non-refundable + 2-adult + room only + free cancellation + 2-adult + breakfast + non-refundable + 2-adult + breakfast + free cancellation + 1-adult + room only / breakfast (filtered out by caller if not needed) +""" + +import asyncio +import logging +import re +from datetime import date +from decimal import Decimal +from typing import List, Optional, Tuple +from urllib.parse import urlparse + +from playwright.async_api import async_playwright, Browser, BrowserContext, Page + +from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus + +logger = logging.getLogger(__name__) + +# JS that extracts all rate plan rows from the room availability table. +# Runs inside the page after the room table has loaded. +_EXTRACT_RATES_JS = """ +() => { + const results = []; + + document.querySelectorAll('[id^="room_type_id_"]').forEach(roomEl => { + const roomId = roomEl.getAttribute('data-room-id') || roomEl.id.replace('room_type_id_', ''); + const roomName = ( + roomEl.querySelector('.hprt-roomtype-icon-link')?.innerText || + roomEl.querySelector('span')?.innerText || '' + ).trim(); + + // Availability count from scarcity indicator ("We have 2 left") + const availText = roomEl.closest('tr') + ?.querySelector('.only_x_left, .thisRoomAvailabilityNew span') + ?.innerText?.trim() || ''; + const availMatch = availText.match(/\\d+/); + const availCount = availMatch ? parseInt(availMatch[0]) : null; + + // Walk sibling rows that belong to this room type + let tr = roomEl.closest('tr'); + while (tr) { + if (tr.classList.contains('js-rt-block-row')) { + const blockId = tr.getAttribute('data-block-id') || ''; + const priceRaw = tr.getAttribute('data-hotel-rounded-price') || ''; + + let fltrs = {}; + try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {} + + // Conditions cell (3rd ) holds meal plan + cancel info + const cells = tr.querySelectorAll('td'); + const condCell = cells.length >= 3 ? cells[2].innerText || '' : ''; + + const breakfastIncluded = condCell.toLowerCase().includes('breakfast'); + const nonRefundable = (fltrs.non_refundable === 1); + + // Free cancellation date: "Free cancellation before DD Month YYYY" + const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i); + const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null; + + // Persons: first ("Max persons: 2" or "Only for 1 guest") + const personsCell = cells.length >= 1 ? cells[0].innerText || '' : ''; + const personsMatch = personsCell.match(/\\d+/); + const maxPersons = personsMatch ? parseInt(personsMatch[0]) : null; + + results.push({ + room_id: roomId, + room_name: roomName, + avail_count: availCount, + block_id: blockId, + price: priceRaw ? parseInt(priceRaw) : null, + breakfast_included: breakfastIncluded, + non_refundable: nonRefundable, + free_cancel_text: freeCancelText, + max_persons: maxPersons, + }); + } + + tr = tr.nextElementSibling; + if (!tr) break; + // Stop at the next room type's header row + if (tr.querySelector('[id^="room_type_id_"]')) break; + } + }); + + return results; +} +""" + + +def _slug_from_url(url: str) -> Optional[str]: + """Extract hotel slug from Booking.com hotel page URL.""" + if not url: + return None + m = re.search(r'/hotel/\w+/([^.]+)\.', url) + return m.group(1) if m else None + + +def _build_hotel_url(hotel_url: str, checkin: date, checkout: date, adults: int) -> str: + base = hotel_url.split('?')[0] + return ( + f"{base}?checkin={checkin}&checkout={checkout}" + f"&group_adults={adults}&no_rooms=1&selected_currency=GBP" + ) + + +class PlaywrightHotelPageBackend(ScraperBackend): + """ + Scrapes individual Booking.com hotel property pages. + + Keeps one browser context alive across multiple hotel+date requests. + Rotates proxy only when a CF/WAF block is detected. + """ + + def __init__(self, proxy_config: dict = None): + self._proxy_config = proxy_config or {} + self._pw = None + self._browser: Optional[Browser] = None + self._context: Optional[BrowserContext] = None + self._requests_on_context = 0 + # Rotate after this many requests even without a block (keeps session fresh) + self._max_requests_per_context = 40 + + def _proxy_enabled(self) -> bool: + return bool(self._proxy_config.get('server')) + + def _proxy_kwargs(self) -> dict: + if not self._proxy_enabled(): + return {} + cfg = self._proxy_config + proxy = {'server': cfg['server']} + if cfg.get('username'): + proxy['username'] = cfg['username'] + if cfg.get('password'): + proxy['password'] = cfg['password'] + return {'proxy': proxy} + + async def _start(self): + if not self._pw: + self._pw = await async_playwright().start() + if not self._browser: + self._browser = await self._pw.chromium.launch( + headless=True, + args=['--no-sandbox', '--disable-dev-shm-usage'], + ) + + async def _get_context(self) -> BrowserContext: + await self._start() + if self._context is None or self._requests_on_context >= self._max_requests_per_context: + if self._context: + try: + await self._context.close() + except Exception: + pass + self._context = await self._browser.new_context( + user_agent=( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/125.0.0.0 Safari/537.36' + ), + **self._proxy_kwargs(), + ) + self._requests_on_context = 0 + logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else "")) + return self._context + + async def _rotate_context(self): + """Force a new browser context (new proxy session).""" + if self._context: + try: + await self._context.close() + except Exception: + pass + self._context = None + self._requests_on_context = 0 + logger.info("Rotated browser context (new proxy session)") + + async def scrape_hotel_page( + self, + hotel_url: str, + check_in: date, + check_out: date, + adults: int = 2, + ) -> ScraperResult: + """ + Load a hotel page and extract all room types + rate plan variants. + + On block detection, rotates proxy and returns blocked=True so the + caller can retry with the fresh context. + """ + page_url = _build_hotel_url(hotel_url, check_in, check_out, adults) + context = await self._get_context() + self._requests_on_context += 1 + page: Optional[Page] = None + + try: + page = await context.new_page() + + # Load the page + page_loaded = False + try: + await page.goto(page_url, wait_until='domcontentloaded', timeout=30000) + page_loaded = True + except Exception as e: + logger.warning(f"goto timeout for {hotel_url} {check_in}: {e}") + + if not page_loaded: + return ScraperResult(success=False, blocked=False, error_message='page load timeout') + + # Wait for the room table (prices render via JS after DOM) + room_table_appeared = False + try: + await page.wait_for_selector('[id^="room_type_id_"]', timeout=15000) + room_table_appeared = True + except Exception: + pass + + if not room_table_appeared: + # Could be sold-out, no-availability, or a block page + content = await page.content() + is_blocked, reason = self.detect_blocking(content) + if is_blocked or len(content) < 20000: + logger.warning(f"Block detected for {hotel_url} {check_in}: {reason or 'small page'}") + await self._rotate_context() + return ScraperResult(success=False, blocked=True, block_reason=reason or 'small page') + + # No availability for this date + return ScraperResult( + success=True, + rates=[RateData( + rate_date=check_in, + availability_status=AvailabilityStatus.SOLD_OUT, + currency='GBP', + )], + ) + + # Extra wait for JS prices to render + await asyncio.sleep(10) + + # Check for explicit "no availability" message + no_avail_el = await page.query_selector('#no_availability_msg') + if no_avail_el and await no_avail_el.is_visible(): + return ScraperResult( + success=True, + rates=[RateData( + rate_date=check_in, + availability_status=AvailabilityStatus.SOLD_OUT, + currency='GBP', + )], + ) + + # Extract all rate plan rows + raw_plans = await page.evaluate(_EXTRACT_RATES_JS) + + if not raw_plans: + return ScraperResult(success=True, rates=[]) + + rates: List[RateData] = [] + for plan in raw_plans: + if plan['price'] is None: + continue + rates.append(RateData( + rate_date=check_in, + availability_status=AvailabilityStatus.AVAILABLE, + rate_gross=Decimal(plan['price']), + currency='GBP', + room_type=plan['room_name'] or None, + breakfast_included=plan['breakfast_included'], + free_cancellation=not plan['non_refundable'], + rooms_left=plan['avail_count'], + rate_plan_id=plan['block_id'] or None, + max_persons=plan['max_persons'], + )) + + logger.info( + f"Hotel page {_slug_from_url(hotel_url)} {check_in}: " + f"{len(raw_plans)} rate plans extracted" + ) + return ScraperResult(success=True, rates=rates) + + except Exception as e: + logger.error(f"scrape_hotel_page error for {hotel_url} {check_in}: {e}") + return ScraperResult(success=False, error_message=str(e)) + finally: + if page: + try: + await page.close() + except Exception: + pass + + async def scrape_location_search(self, *args, **kwargs) -> ScraperResult: + """Not used by this backend — hotel pages replace location search.""" + raise NotImplementedError("PlaywrightHotelPageBackend does not support location search") + + async def close(self): + if self._context: + try: + await self._context.close() + except Exception: + pass + self._context = None + if self._browser: + try: + await self._browser.close() + except Exception: + pass + self._browser = None + if self._pw: + try: + await self._pw.stop() + except Exception: + pass + self._pw = None