# instead of for the room_type_id_ element — both caused under-counting. _EXTRACT_RATES_JS = """ () => { const results = []; const roomTypeNames = {}; const rows = document.querySelectorAll( '#available_rooms tbody tr:not([data-is-room-upgrade])' ); for (const tr of rows) { const blockId = tr.getAttribute('data-block-id') || ''; if (!blockId) continue; const blockParts = blockId.split('_'); if (blockParts.length < 2) continue; const roomId = blockParts[0]; // Room name: the first row for each room type carries the room_type_id_ element. // Subsequent rows for the same room type don't — reuse stored name (same as scrapy). const roomNameEl = tr.querySelector('[id^="room_type_id_"]'); if (roomNameEl) { const name = ( roomNameEl.querySelector('.hprt-roomtype-icon-link')?.innerText || roomNameEl.querySelector('span')?.innerText || roomNameEl.innerText || '' ).trim(); if (name) roomTypeNames[roomId] = name; } const roomName = roomTypeNames[roomId] || ''; const priceRaw = tr.getAttribute('data-hotel-rounded-price') || ''; let fltrs = {}; try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {} const cells = tr.querySelectorAll('td'); // Extract condition detail lines using the same text-search approach as the // original scrapy spider: grab

items from the row (Booking.com renders // meal plan, cancellation policy and payment method each as a separate

), // then search each line for known keywords. No match → null, not assumed false. const liTexts = Array.from(tr.querySelectorAll('li')) .map(li => (li.innerText || '').trim()) .filter(Boolean); const detailLines = liTexts.length ? liTexts : (tr.innerText || '').split('\\n').map(l => l.trim()).filter(l => l.length > 2 && l.length < 200); let breakfastText = null; let cancelText = null; let paymentText = null; for (const line of detailLines) { const ll = line.toLowerCase(); if (!breakfastText && ll.includes('breakfast')) breakfastText = line; if (!cancelText && (ll.includes('free cancellation') || ll.includes('non-refundable') || ll.includes('total cost to cancel') || ll.includes('fully chargeable') || ll.includes('partially refundable'))) cancelText = line; if (!paymentText && (ll.includes('no prepayment') || ll.includes('pay at the property') || ll.includes('pay the property') || ll.includes('pay online') || ll.includes('pay now'))) paymentText = line; } // Cross-check with structured data-fltrs — use as fallback when text is absent const fltrsBreakfast = fltrs.mealplan === 1 || fltrs.breakfast_included === 1; const fltrsNonRefundable = fltrs.non_refundable === 1; if (!breakfastText && fltrsBreakfast) breakfastText = 'Breakfast included'; if (!cancelText && fltrsNonRefundable) cancelText = 'Non-refundable'; if (!cancelText && fltrs.non_refundable === 0) cancelText = 'Free cancellation'; // Derive booleans — text is authoritative, fltrs as fallback let breakfastIncluded = null; if (breakfastText) { breakfastIncluded = breakfastText.toLowerCase().includes('included') || fltrsBreakfast; } else if (fltrsBreakfast) { breakfastIncluded = true; } let freeCancellation = null; if (cancelText) { freeCancellation = cancelText.toLowerCase().includes('free cancellation'); } else if (fltrsNonRefundable) { freeCancellation = false; } let noPrepayment = null; if (paymentText) { const pll = paymentText.toLowerCase(); noPrepayment = pll.includes('no prepayment') || pll.includes('pay at the property') || pll.includes('pay the property'); } // Max persons: read from visible span text (same as original scrapy spider). // block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy. let maxPersons = null; for (const span of tr.querySelectorAll('span')) { const t = span.innerText || ''; const m = t.match(/Max persons?:?\\s*(\\d+)/i); if (m) { maxPersons = parseInt(m[1]); break; } } // Also try occupancy icon count (aria-label or title with "X adults") if (maxPersons === null) { const occMatch = tr.innerHTML.match(/title="(\\d+) adults?"/i) || tr.innerHTML.match(/aria-label="(\\d+) adults?"/i); if (occMatch) maxPersons = parseInt(occMatch[1]); } // Scarcity text ("We have 2 left") as fallback for availability const availText = tr.querySelector('.only_x_left, .thisRoomAvailabilityNew span') ?.innerText?.trim() || ''; const availMatch = availText.match(/\\d+/); let qtyAvailable = availMatch ? parseInt(availMatch[0]) : null; // Quantity dropdown in last : options 0..N where N = available qty (capped at 10) const qtyCell = cells.length >= 2 ? cells[cells.length - 1] : null; if (qtyCell) { const sel = qtyCell.querySelector('select'); if (sel) { const vals = Array.from(sel.options) .map(o => parseInt(o.value)) .filter(v => !isNaN(v) && v > 0); if (vals.length > 0) qtyAvailable = Math.max(...vals); } else { const nums = (qtyCell.innerText || '').match(/^(\\d+)/gm); if (nums && nums.length > 0) { const parsed = nums.map(n => parseInt(n)).filter(v => v > 0); if (parsed.length > 0) qtyAvailable = Math.max(...parsed); } } } results.push({ room_id: roomId, room_name: roomName, avail_count: qtyAvailable, block_id: blockId, price: priceRaw ? parseInt(priceRaw) : null, breakfast_text: breakfastText, cancel_text: cancelText, payment_text: paymentText, breakfast_included: breakfastIncluded, free_cancellation: freeCancellation, no_prepayment: noPrepayment, max_persons: maxPersons, }); } return results; } """ def _slug_from_url(url: str) -> Optional[str]: """Extract hotel slug from Booking.com hotel page URL.""" if not url: return None m = re.search(r'/hotel/\w+/([^.]+)\.', url) return m.group(1) if m else None def _build_hotel_url(hotel_url: str, checkin: date, checkout: date, adults: int) -> str: base = hotel_url.split('?')[0] return ( f"{base}?checkin={checkin}&checkout={checkout}" f"&group_adults={adults}&no_rooms=1&selected_currency=GBP" ) class PlaywrightHotelPageBackend(ScraperBackend): """ Scrapes individual Booking.com hotel property pages. Keeps one browser context alive across multiple hotel+date requests. Rotates proxy only when a CF/WAF block is detected. """ def __init__(self, proxy_config: dict = None): self._proxy_config = proxy_config or {} self._pw = None self._browser: Optional[Browser] = None self._context: Optional[BrowserContext] = None self._requests_on_context = 0 # Rotate after this many requests even without a block (keeps session fresh) self._max_requests_per_context = 40 def _proxy_enabled(self) -> bool: return proxy_util.is_enabled(self._proxy_config) def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict: # Credentials embedded in URL — separate username/password fields cause # a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment). proxy = proxy_util.playwright_proxy(self._proxy_config, session_id) return {'proxy': proxy} if proxy else {} async def _start(self): if not self._pw: self._pw = await async_playwright().start() if not self._browser: self._browser = await self._pw.chromium.launch( headless=True, args=[ '--no-sandbox', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled', '--disable-infobars', '--disable-extensions', '--blink-settings=imagesEnabled=false', ], ) async def _get_context(self) -> BrowserContext: await self._start() if self._context is None or self._requests_on_context >= self._max_requests_per_context: if self._context: try: await self._context.close() except Exception: pass session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None ua = random.choice(_USER_AGENTS) viewport = random.choice(_VIEWPORTS) self._context = await self._browser.new_context( user_agent=ua, viewport=viewport, locale='en-GB', timezone_id='Europe/London', **self._proxy_kwargs(session_id), ) await self._context.add_init_script(_STEALTH_SCRIPT) self._requests_on_context = 0 logger.debug( f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}" + (f" proxy_session={session_id}" if session_id else "") ) return self._context async def _rotate_context(self): """Force a new browser context (new proxy session).""" if self._context: try: await self._context.close() except Exception: pass self._context = None self._requests_on_context = 0 logger.info("Rotated browser context (new proxy session)") async def scrape_hotel_page( self, hotel_url: str, check_in: date, check_out: date, adults: int = 2, ) -> ScraperResult: """ Load a hotel page and extract all room types + rate plan variants. On block detection, rotates proxy and returns blocked=True so the caller can retry with the fresh context. """ page_url = _build_hotel_url(hotel_url, check_in, check_out, adults) context = await self._get_context() self._requests_on_context += 1 page: Optional[Page] = None try: page = await context.new_page() # Load the page page_loaded = False try: await page.goto(page_url, wait_until='domcontentloaded', timeout=30000) page_loaded = True except Exception as e: logger.warning(f"goto timeout for {hotel_url} {check_in}: {e}") if not page_loaded: # Treat as a retryable block — DataImpulse endpoints are intermittently # broken (EOF after CONNECT); rotating to a fresh context hits a different # endpoint and usually succeeds within 1-2 retries. await self._rotate_context() return ScraperResult(success=False, blocked=True, block_reason='page load timeout') # Wait for the room table (prices render via JS after DOM) room_table_appeared = False try: await page.wait_for_selector('[id^="room_type_id_"]', timeout=15000) room_table_appeared = True except Exception: pass if not room_table_appeared: # Could be sold-out, no-availability, or a block page content = await page.content() is_blocked, reason = self.detect_blocking(content) if is_blocked or len(content) < 20000: logger.warning(f"Block detected for {hotel_url} {check_in}: {reason or 'small page'}") await self._rotate_context() return ScraperResult(success=False, blocked=True, block_reason=reason or 'small page') # No availability for this date return ScraperResult( success=True, rates=[RateData( rate_date=check_in, availability_status=AvailabilityStatus.SOLD_OUT, currency='GBP', )], ) # Extra wait for JS prices to render await asyncio.sleep(10) # Check for explicit "no availability" message no_avail_el = await page.query_selector('#no_availability_msg') if no_avail_el and await no_avail_el.is_visible(): return ScraperResult( success=True, rates=[RateData( rate_date=check_in, availability_status=AvailabilityStatus.SOLD_OUT, currency='GBP', )], ) # Extract all rate plan rows raw_plans = await page.evaluate(_EXTRACT_RATES_JS) if not raw_plans: return ScraperResult(success=True, rates=[]) rates: List[RateData] = [] for plan in raw_plans: if plan['price'] is None: continue rates.append(RateData( rate_date=check_in, availability_status=AvailabilityStatus.AVAILABLE, rate_gross=Decimal(plan['price']), currency='GBP', room_type=plan['room_name'] or None, breakfast_included=plan['breakfast_included'], # True/False/None free_cancellation=plan['free_cancellation'], # True/False/None no_prepayment=plan['no_prepayment'], # True/False/None rooms_left=plan['avail_count'], available_qty=plan['avail_count'], rate_plan_id=plan['block_id'] or None, max_persons=plan['max_persons'], breakfast_text=plan['breakfast_text'], cancel_text=plan['cancel_text'], payment_text=plan['payment_text'], )) logger.info( f"Hotel page {_slug_from_url(hotel_url)} {check_in}: " f"{len(raw_plans)} rate plans extracted" ) return ScraperResult(success=True, rates=rates) except Exception as e: logger.error(f"scrape_hotel_page error for {hotel_url} {check_in}: {e}") return ScraperResult(success=False, error_message=str(e)) finally: if page: try: await page.close() except Exception: pass async def scrape_location_search(self, *args, **kwargs) -> ScraperResult: """Not used by this backend — hotel pages replace location search.""" raise NotImplementedError("PlaywrightHotelPageBackend does not support location search") async def close(self): if self._context: try: await self._context.close() except Exception: pass self._context = None if self._browser: try: await self._browser.close() except Exception: pass self._browser = None if self._pw: try: await self._pw.stop() except Exception: pass self._pw = None