jtricerolph
38b0c36923
Original scrapy approach: extract <li> condition lines from each rate plan row and text-search for known keywords. No match = null (unknown), not assumed false. Adds breakfast_text, cancel_text, payment_text columns to booking_com_rates. Booleans now nullable (null = not mentioned, true/false = explicit signal). JS extraction searches all <li> items in the row (falls back to newline-split innerText if none). Breakfast: 'breakfast' keyword. Cancel: 'free cancellation', 'non-refundable', 'total cost to cancel', 'fully chargeable'. Payment: 'no prepayment', 'pay at the property', 'pay online'. data-fltrs used as fallback. API snapshot endpoint now returns breakfast/cancel/payment text strings. Old boolean-only rows degrade gracefully to derived labels. Modal plan rows replace the fixed meal|cancel|price column layout with a single stacked conditions cell: 0-3 lines depending on what the page actually shows. Breakfast green when 'included', cancel green when 'Free cancellation…'. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
467 lines
19 KiB
Python
467 lines
19 KiB
Python
"""
|
|
Booking.com Hotel Page Scraper Backend
|
|
|
|
Scrapes individual hotel property pages instead of search results.
|
|
Returns all room types and rate plan variants per hotel per date.
|
|
|
|
Advantages over search-results approach:
|
|
- Individual hotel pages are not Cloudflare-protected (search results are)
|
|
- Captures every room type, rate plan, meal plan, and availability count
|
|
- Proxy reuse: one residential IP stays valid across many hotel pages
|
|
(looks like a human browsing properties), rotates only when blocked
|
|
|
|
Rate plan variants per room type (typical):
|
|
2-adult + room only + non-refundable
|
|
2-adult + room only + free cancellation
|
|
2-adult + breakfast + non-refundable
|
|
2-adult + breakfast + free cancellation
|
|
1-adult + room only / breakfast (filtered out by caller if not needed)
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import re
|
|
import uuid
|
|
from datetime import date
|
|
from decimal import Decimal
|
|
from typing import List, Optional, Tuple
|
|
from urllib.parse import urlparse
|
|
|
|
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
|
|
|
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
|
|
from services import proxy as proxy_util
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_USER_AGENTS = [
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0',
|
|
]
|
|
|
|
_VIEWPORTS = [
|
|
{'width': 1920, 'height': 1080},
|
|
{'width': 1366, 'height': 768},
|
|
{'width': 1440, 'height': 900},
|
|
{'width': 1536, 'height': 864},
|
|
{'width': 1280, 'height': 800},
|
|
{'width': 1600, 'height': 900},
|
|
]
|
|
|
|
# Injected into every new context to mask headless/webdriver signals.
|
|
_STEALTH_SCRIPT = """
|
|
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
|
Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
|
|
Object.defineProperty(navigator, 'languages', {get: () => ['en-GB', 'en']});
|
|
window.chrome = {runtime: {}};
|
|
"""
|
|
|
|
# JS that extracts all rate plan rows from the room availability table.
|
|
# Runs inside the page after the room table has loaded.
|
|
#
|
|
# Iterates ALL tbody rows (matching the original scrapy spider approach) rather than
|
|
# walking siblings from room-type anchors. This handles hotels where the first rate
|
|
# row of a room type lacks js-rt-block-row, or where the room header uses a <div>
|
|
# instead of <a> for the room_type_id_ element — both caused under-counting.
|
|
_EXTRACT_RATES_JS = """
|
|
() => {
|
|
const results = [];
|
|
const roomTypeNames = {};
|
|
|
|
const rows = document.querySelectorAll(
|
|
'#available_rooms tbody tr:not([data-is-room-upgrade])'
|
|
);
|
|
|
|
for (const tr of rows) {
|
|
const blockId = tr.getAttribute('data-block-id') || '';
|
|
if (!blockId) continue;
|
|
|
|
const blockParts = blockId.split('_');
|
|
if (blockParts.length < 2) continue;
|
|
|
|
const roomId = blockParts[0];
|
|
|
|
// Room name: the first row for each room type carries the room_type_id_ element.
|
|
// Subsequent rows for the same room type don't — reuse stored name (same as scrapy).
|
|
const roomNameEl = tr.querySelector('[id^="room_type_id_"]');
|
|
if (roomNameEl) {
|
|
const name = (
|
|
roomNameEl.querySelector('.hprt-roomtype-icon-link')?.innerText ||
|
|
roomNameEl.querySelector('span')?.innerText ||
|
|
roomNameEl.innerText ||
|
|
''
|
|
).trim();
|
|
if (name) roomTypeNames[roomId] = name;
|
|
}
|
|
const roomName = roomTypeNames[roomId] || '';
|
|
|
|
const priceRaw = tr.getAttribute('data-hotel-rounded-price') || '';
|
|
|
|
let fltrs = {};
|
|
try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {}
|
|
|
|
const cells = tr.querySelectorAll('td');
|
|
|
|
// Extract condition detail lines using the same text-search approach as the
|
|
// original scrapy spider: grab <li> items from the row (Booking.com renders
|
|
// meal plan, cancellation policy and payment method each as a separate <li>),
|
|
// then search each line for known keywords. No match → null, not assumed false.
|
|
const liTexts = Array.from(tr.querySelectorAll('li'))
|
|
.map(li => (li.innerText || '').trim())
|
|
.filter(Boolean);
|
|
const detailLines = liTexts.length
|
|
? liTexts
|
|
: (tr.innerText || '').split('\\n').map(l => l.trim()).filter(l => l.length > 2 && l.length < 200);
|
|
|
|
let breakfastText = null;
|
|
let cancelText = null;
|
|
let paymentText = null;
|
|
|
|
for (const line of detailLines) {
|
|
const ll = line.toLowerCase();
|
|
if (!breakfastText && ll.includes('breakfast')) breakfastText = line;
|
|
if (!cancelText && (ll.includes('free cancellation') || ll.includes('non-refundable')
|
|
|| ll.includes('total cost to cancel') || ll.includes('fully chargeable')
|
|
|| ll.includes('partially refundable'))) cancelText = line;
|
|
if (!paymentText && (ll.includes('no prepayment') || ll.includes('pay at the property')
|
|
|| ll.includes('pay the property') || ll.includes('pay online')
|
|
|| ll.includes('pay now'))) paymentText = line;
|
|
}
|
|
|
|
// Cross-check with structured data-fltrs — use as fallback when text is absent
|
|
const fltrsBreakfast = fltrs.mealplan === 1 || fltrs.breakfast_included === 1;
|
|
const fltrsNonRefundable = fltrs.non_refundable === 1;
|
|
|
|
if (!breakfastText && fltrsBreakfast) breakfastText = 'Breakfast included';
|
|
if (!cancelText && fltrsNonRefundable) cancelText = 'Non-refundable';
|
|
if (!cancelText && fltrs.non_refundable === 0) cancelText = 'Free cancellation';
|
|
|
|
// Derive booleans — text is authoritative, fltrs as fallback
|
|
let breakfastIncluded = null;
|
|
if (breakfastText) {
|
|
breakfastIncluded = breakfastText.toLowerCase().includes('included') || fltrsBreakfast;
|
|
} else if (fltrsBreakfast) {
|
|
breakfastIncluded = true;
|
|
}
|
|
|
|
let freeCancellation = null;
|
|
if (cancelText) {
|
|
freeCancellation = cancelText.toLowerCase().includes('free cancellation');
|
|
} else if (fltrsNonRefundable) {
|
|
freeCancellation = false;
|
|
}
|
|
|
|
let noPrepayment = null;
|
|
if (paymentText) {
|
|
const pll = paymentText.toLowerCase();
|
|
noPrepayment = pll.includes('no prepayment') || pll.includes('pay at the property')
|
|
|| pll.includes('pay the property');
|
|
}
|
|
|
|
// Max persons: read from visible span text (same as original scrapy spider).
|
|
// block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy.
|
|
let maxPersons = null;
|
|
for (const span of tr.querySelectorAll('span')) {
|
|
const t = span.innerText || '';
|
|
const m = t.match(/Max persons?:?\\s*(\\d+)/i);
|
|
if (m) { maxPersons = parseInt(m[1]); break; }
|
|
}
|
|
// Also try occupancy icon count (aria-label or title with "X adults")
|
|
if (maxPersons === null) {
|
|
const occMatch = tr.innerHTML.match(/title="(\\d+) adults?"/i)
|
|
|| tr.innerHTML.match(/aria-label="(\\d+) adults?"/i);
|
|
if (occMatch) maxPersons = parseInt(occMatch[1]);
|
|
}
|
|
|
|
// Scarcity text ("We have 2 left") as fallback for availability
|
|
const availText = tr.querySelector('.only_x_left, .thisRoomAvailabilityNew span')
|
|
?.innerText?.trim() || '';
|
|
const availMatch = availText.match(/\\d+/);
|
|
let qtyAvailable = availMatch ? parseInt(availMatch[0]) : null;
|
|
|
|
// Quantity dropdown in last <td>: options 0..N where N = available qty (capped at 10)
|
|
const qtyCell = cells.length >= 2 ? cells[cells.length - 1] : null;
|
|
if (qtyCell) {
|
|
const sel = qtyCell.querySelector('select');
|
|
if (sel) {
|
|
const vals = Array.from(sel.options)
|
|
.map(o => parseInt(o.value))
|
|
.filter(v => !isNaN(v) && v > 0);
|
|
if (vals.length > 0) qtyAvailable = Math.max(...vals);
|
|
} else {
|
|
const nums = (qtyCell.innerText || '').match(/^(\\d+)/gm);
|
|
if (nums && nums.length > 0) {
|
|
const parsed = nums.map(n => parseInt(n)).filter(v => v > 0);
|
|
if (parsed.length > 0) qtyAvailable = Math.max(...parsed);
|
|
}
|
|
}
|
|
}
|
|
|
|
results.push({
|
|
room_id: roomId,
|
|
room_name: roomName,
|
|
avail_count: qtyAvailable,
|
|
block_id: blockId,
|
|
price: priceRaw ? parseInt(priceRaw) : null,
|
|
breakfast_text: breakfastText,
|
|
cancel_text: cancelText,
|
|
payment_text: paymentText,
|
|
breakfast_included: breakfastIncluded,
|
|
free_cancellation: freeCancellation,
|
|
no_prepayment: noPrepayment,
|
|
max_persons: maxPersons,
|
|
});
|
|
}
|
|
|
|
return results;
|
|
}
|
|
"""
|
|
|
|
|
|
def _slug_from_url(url: str) -> Optional[str]:
|
|
"""Extract hotel slug from Booking.com hotel page URL."""
|
|
if not url:
|
|
return None
|
|
m = re.search(r'/hotel/\w+/([^.]+)\.', url)
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def _build_hotel_url(hotel_url: str, checkin: date, checkout: date, adults: int) -> str:
|
|
base = hotel_url.split('?')[0]
|
|
return (
|
|
f"{base}?checkin={checkin}&checkout={checkout}"
|
|
f"&group_adults={adults}&no_rooms=1&selected_currency=GBP"
|
|
)
|
|
|
|
|
|
class PlaywrightHotelPageBackend(ScraperBackend):
|
|
"""
|
|
Scrapes individual Booking.com hotel property pages.
|
|
|
|
Keeps one browser context alive across multiple hotel+date requests.
|
|
Rotates proxy only when a CF/WAF block is detected.
|
|
"""
|
|
|
|
def __init__(self, proxy_config: dict = None):
|
|
self._proxy_config = proxy_config or {}
|
|
self._pw = None
|
|
self._browser: Optional[Browser] = None
|
|
self._context: Optional[BrowserContext] = None
|
|
self._requests_on_context = 0
|
|
# Rotate after this many requests even without a block (keeps session fresh)
|
|
self._max_requests_per_context = 40
|
|
|
|
def _proxy_enabled(self) -> bool:
|
|
return proxy_util.is_enabled(self._proxy_config)
|
|
|
|
def _proxy_kwargs(self, session_id: Optional[str] = None) -> dict:
|
|
# Credentials embedded in URL — separate username/password fields cause
|
|
# a 407 round-trip that takes ~14s on DataImpulse (see proxy.py comment).
|
|
proxy = proxy_util.playwright_proxy(self._proxy_config, session_id)
|
|
return {'proxy': proxy} if proxy else {}
|
|
|
|
async def _start(self):
|
|
if not self._pw:
|
|
self._pw = await async_playwright().start()
|
|
if not self._browser:
|
|
self._browser = await self._pw.chromium.launch(
|
|
headless=True,
|
|
args=[
|
|
'--no-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-blink-features=AutomationControlled',
|
|
'--disable-infobars',
|
|
'--disable-extensions',
|
|
'--blink-settings=imagesEnabled=false',
|
|
],
|
|
)
|
|
|
|
async def _get_context(self) -> BrowserContext:
|
|
await self._start()
|
|
if self._context is None or self._requests_on_context >= self._max_requests_per_context:
|
|
if self._context:
|
|
try:
|
|
await self._context.close()
|
|
except Exception:
|
|
pass
|
|
session_id = uuid.uuid4().hex[:12] if self._proxy_enabled() else None
|
|
ua = random.choice(_USER_AGENTS)
|
|
viewport = random.choice(_VIEWPORTS)
|
|
self._context = await self._browser.new_context(
|
|
user_agent=ua,
|
|
viewport=viewport,
|
|
locale='en-GB',
|
|
timezone_id='Europe/London',
|
|
**self._proxy_kwargs(session_id),
|
|
)
|
|
await self._context.add_init_script(_STEALTH_SCRIPT)
|
|
self._requests_on_context = 0
|
|
logger.debug(
|
|
f"Opened new browser context ua=…{ua[-30:]} viewport={viewport['width']}x{viewport['height']}"
|
|
+ (f" proxy_session={session_id}" if session_id else "")
|
|
)
|
|
return self._context
|
|
|
|
async def _rotate_context(self):
|
|
"""Force a new browser context (new proxy session)."""
|
|
if self._context:
|
|
try:
|
|
await self._context.close()
|
|
except Exception:
|
|
pass
|
|
self._context = None
|
|
self._requests_on_context = 0
|
|
logger.info("Rotated browser context (new proxy session)")
|
|
|
|
async def scrape_hotel_page(
|
|
self,
|
|
hotel_url: str,
|
|
check_in: date,
|
|
check_out: date,
|
|
adults: int = 2,
|
|
) -> ScraperResult:
|
|
"""
|
|
Load a hotel page and extract all room types + rate plan variants.
|
|
|
|
On block detection, rotates proxy and returns blocked=True so the
|
|
caller can retry with the fresh context.
|
|
"""
|
|
page_url = _build_hotel_url(hotel_url, check_in, check_out, adults)
|
|
context = await self._get_context()
|
|
self._requests_on_context += 1
|
|
page: Optional[Page] = None
|
|
|
|
try:
|
|
page = await context.new_page()
|
|
|
|
# Load the page
|
|
page_loaded = False
|
|
try:
|
|
await page.goto(page_url, wait_until='domcontentloaded', timeout=30000)
|
|
page_loaded = True
|
|
except Exception as e:
|
|
logger.warning(f"goto timeout for {hotel_url} {check_in}: {e}")
|
|
|
|
if not page_loaded:
|
|
# Treat as a retryable block — DataImpulse endpoints are intermittently
|
|
# broken (EOF after CONNECT); rotating to a fresh context hits a different
|
|
# endpoint and usually succeeds within 1-2 retries.
|
|
await self._rotate_context()
|
|
return ScraperResult(success=False, blocked=True, block_reason='page load timeout')
|
|
|
|
# Wait for the room table (prices render via JS after DOM)
|
|
room_table_appeared = False
|
|
try:
|
|
await page.wait_for_selector('[id^="room_type_id_"]', timeout=15000)
|
|
room_table_appeared = True
|
|
except Exception:
|
|
pass
|
|
|
|
if not room_table_appeared:
|
|
# Could be sold-out, no-availability, or a block page
|
|
content = await page.content()
|
|
is_blocked, reason = self.detect_blocking(content)
|
|
if is_blocked or len(content) < 20000:
|
|
logger.warning(f"Block detected for {hotel_url} {check_in}: {reason or 'small page'}")
|
|
await self._rotate_context()
|
|
return ScraperResult(success=False, blocked=True, block_reason=reason or 'small page')
|
|
|
|
# No availability for this date
|
|
return ScraperResult(
|
|
success=True,
|
|
rates=[RateData(
|
|
rate_date=check_in,
|
|
availability_status=AvailabilityStatus.SOLD_OUT,
|
|
currency='GBP',
|
|
)],
|
|
)
|
|
|
|
# Extra wait for JS prices to render
|
|
await asyncio.sleep(10)
|
|
|
|
# Check for explicit "no availability" message
|
|
no_avail_el = await page.query_selector('#no_availability_msg')
|
|
if no_avail_el and await no_avail_el.is_visible():
|
|
return ScraperResult(
|
|
success=True,
|
|
rates=[RateData(
|
|
rate_date=check_in,
|
|
availability_status=AvailabilityStatus.SOLD_OUT,
|
|
currency='GBP',
|
|
)],
|
|
)
|
|
|
|
# Extract all rate plan rows
|
|
raw_plans = await page.evaluate(_EXTRACT_RATES_JS)
|
|
|
|
if not raw_plans:
|
|
return ScraperResult(success=True, rates=[])
|
|
|
|
rates: List[RateData] = []
|
|
for plan in raw_plans:
|
|
if plan['price'] is None:
|
|
continue
|
|
rates.append(RateData(
|
|
rate_date=check_in,
|
|
availability_status=AvailabilityStatus.AVAILABLE,
|
|
rate_gross=Decimal(plan['price']),
|
|
currency='GBP',
|
|
room_type=plan['room_name'] or None,
|
|
breakfast_included=plan['breakfast_included'], # True/False/None
|
|
free_cancellation=plan['free_cancellation'], # True/False/None
|
|
no_prepayment=plan['no_prepayment'], # True/False/None
|
|
rooms_left=plan['avail_count'],
|
|
available_qty=plan['avail_count'],
|
|
rate_plan_id=plan['block_id'] or None,
|
|
max_persons=plan['max_persons'],
|
|
breakfast_text=plan['breakfast_text'],
|
|
cancel_text=plan['cancel_text'],
|
|
payment_text=plan['payment_text'],
|
|
))
|
|
|
|
logger.info(
|
|
f"Hotel page {_slug_from_url(hotel_url)} {check_in}: "
|
|
f"{len(raw_plans)} rate plans extracted"
|
|
)
|
|
return ScraperResult(success=True, rates=rates)
|
|
|
|
except Exception as e:
|
|
logger.error(f"scrape_hotel_page error for {hotel_url} {check_in}: {e}")
|
|
return ScraperResult(success=False, error_message=str(e))
|
|
finally:
|
|
if page:
|
|
try:
|
|
await page.close()
|
|
except Exception:
|
|
pass
|
|
|
|
async def scrape_location_search(self, *args, **kwargs) -> ScraperResult:
|
|
"""Not used by this backend — hotel pages replace location search."""
|
|
raise NotImplementedError("PlaywrightHotelPageBackend does not support location search")
|
|
|
|
async def close(self):
|
|
if self._context:
|
|
try:
|
|
await self._context.close()
|
|
except Exception:
|
|
pass
|
|
self._context = None
|
|
if self._browser:
|
|
try:
|
|
await self._browser.close()
|
|
except Exception:
|
|
pass
|
|
self._browser = None
|
|
if self._pw:
|
|
try:
|
|
await self._pw.stop()
|
|
except Exception:
|
|
pass
|
|
self._pw = None
|