Add hotel-page scraper backend with full room/rate plan extraction
Switches booking.com scraping from search-results page (Cloudflare-targeted) to individual hotel property pages (not CF-protected). Each page load returns all room types, all rate plan variants (room-only/B&B × refundable/non-ref × 1-2 adults), and availability counts. Key changes: - New PlaywrightHotelPageBackend: proxy reuse until block, rotate on CF/WAF - booking_scraper.py: _run_hotel_page_scrape(), scrape_hotel_date(), _scrape_hotels_concurrent() — sharded by hotel so one proxy session covers all dates for one hotel (looks human) - schema.sql: ADD COLUMN rate_plan_id, max_persons on booking_com_rates - competitors API: filter max_persons=2, order by rate_gross ASC as tiebreaker so DISTINCT ON returns cheapest 2-adult rate from latest batch Enable via Settings → Scraper Backend → playwright_hotel_page Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
b10b2f5990
commit
bb37fcf501
6 changed files with 608 additions and 8 deletions
|
|
@ -25,7 +25,10 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from database import SyncSessionLocal
|
||||
from services import proxy as proxy_util
|
||||
from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
|
||||
from .scraper_backends import (
|
||||
ScraperBackend, PlaywrightLocalBackend, PlaywrightHotelPageBackend,
|
||||
HotelData, RateData, AvailabilityStatus,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -94,12 +97,17 @@ def get_scraper_backend(db: Session) -> ScraperBackend:
|
|||
# Future: Apify backend
|
||||
raise NotImplementedError("Apify backend not yet implemented")
|
||||
|
||||
proxy_cfg = proxy_util.load_config(db)
|
||||
|
||||
if backend_type == 'playwright_hotel_page':
|
||||
return PlaywrightHotelPageBackend(proxy_config=proxy_cfg)
|
||||
|
||||
if backend_type not in ('playwright_local', 'playwright_proxy'):
|
||||
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
|
||||
|
||||
# Proxy config resolved by the shared module: system_config is authoritative
|
||||
# when booking_proxy_enabled is set, else BOOKING_PROXY_* env ({} = direct).
|
||||
return PlaywrightLocalBackend(proxy_config=proxy_util.load_config(db))
|
||||
return PlaywrightLocalBackend(proxy_config=proxy_cfg)
|
||||
|
||||
|
||||
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
|
||||
|
|
@ -189,9 +197,11 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
|
|||
text("""
|
||||
INSERT INTO booking_com_rates
|
||||
(hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
|
||||
breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id)
|
||||
breakfast_included, free_cancellation, no_prepayment, rooms_left,
|
||||
rate_plan_id, max_persons, scrape_batch_id)
|
||||
VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
|
||||
:breakfast, :cancel, :prepay, :rooms_left, :batch_id)
|
||||
:breakfast, :cancel, :prepay, :rooms_left,
|
||||
:rate_plan_id, :max_persons, :batch_id)
|
||||
"""),
|
||||
{
|
||||
'hotel_id': hotel_id,
|
||||
|
|
@ -204,6 +214,8 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
|
|||
'cancel': rate.free_cancellation,
|
||||
'prepay': rate.no_prepayment,
|
||||
'rooms_left': rate.rooms_left,
|
||||
'rate_plan_id': rate.rate_plan_id,
|
||||
'max_persons': rate.max_persons,
|
||||
'batch_id': str(batch_id),
|
||||
}
|
||||
)
|
||||
|
|
@ -282,6 +294,69 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
|
|||
return len(cleaned)
|
||||
|
||||
|
||||
def get_active_hotels(db: Session) -> List[Dict[str, Any]]:
|
||||
"""Return all active hotels that have a booking_com_url (needed for hotel-page scraping)."""
|
||||
rows = db.execute(
|
||||
text("""
|
||||
SELECT id, booking_com_id, name, booking_com_url
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE AND booking_com_url IS NOT NULL
|
||||
ORDER BY display_order, id
|
||||
""")
|
||||
).fetchall()
|
||||
return [dict(r._mapping) for r in rows]
|
||||
|
||||
|
||||
async def scrape_hotel_date(
|
||||
db: Session,
|
||||
hotel: Dict[str, Any],
|
||||
rate_date: date,
|
||||
backend: 'PlaywrightHotelPageBackend',
|
||||
batch_id: uuid.UUID,
|
||||
adults: int = 2,
|
||||
max_retries: int = 2,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Scrape all rate plans for one hotel on one date via the hotel page backend.
|
||||
|
||||
On block detection, the backend rotates its proxy context automatically.
|
||||
We retry up to max_retries times to handle the rotation.
|
||||
"""
|
||||
check_out = rate_date + timedelta(days=1)
|
||||
hotel_url = hotel['booking_com_url']
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
result = await backend.scrape_hotel_page(hotel_url, rate_date, check_out, adults)
|
||||
|
||||
if result.success:
|
||||
rates_saved = 0
|
||||
for rate in result.rates:
|
||||
rate.rate_date = rate_date # ensure date is set
|
||||
try:
|
||||
save_rate(db, rate, hotel['id'], batch_id)
|
||||
rates_saved += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Error saving rate plan for {hotel['name']} {rate_date}: {e}")
|
||||
db.rollback()
|
||||
db.commit()
|
||||
return {'success': True, 'blocked': False, 'rates_count': rates_saved}
|
||||
|
||||
if result.blocked and attempt < max_retries:
|
||||
logger.info(
|
||||
f"Hotel {hotel['name']} {rate_date} blocked (attempt {attempt + 1}), retrying…"
|
||||
)
|
||||
continue
|
||||
|
||||
return {
|
||||
'success': False,
|
||||
'blocked': result.blocked,
|
||||
'error': result.error_message,
|
||||
'rates_count': 0,
|
||||
}
|
||||
|
||||
return {'success': False, 'blocked': True, 'rates_count': 0}
|
||||
|
||||
|
||||
async def scrape_date(
|
||||
db: Session,
|
||||
rate_date: date,
|
||||
|
|
@ -438,6 +513,13 @@ def get_scraper_concurrency(db: Session) -> int:
|
|||
return 6
|
||||
|
||||
|
||||
def _is_hotel_page_mode(db: Session) -> bool:
|
||||
row = db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'")
|
||||
).fetchone()
|
||||
return (row and row.config_value) == 'playwright_hotel_page'
|
||||
|
||||
|
||||
def _effective_concurrency(db: Session, n_jobs: int) -> int:
|
||||
"""Clamp configured concurrency to the workload, and force serial when the
|
||||
proxy is off — N workers would share one IP and hammer it, worse than 1."""
|
||||
|
|
@ -451,6 +533,82 @@ def _effective_concurrency(db: Session, n_jobs: int) -> int:
|
|||
return max(1, min(configured, n_jobs))
|
||||
|
||||
|
||||
async def _scrape_hotels_concurrent(
|
||||
hotel_date_jobs: List[Tuple[Dict[str, Any], date]],
|
||||
concurrency: int,
|
||||
batch_id: uuid.UUID,
|
||||
adults: int = 2,
|
||||
) -> Dict[str, int]:
|
||||
"""
|
||||
Scrape (hotel, date) pairs using the hotel-page backend.
|
||||
|
||||
Jobs are sharded by HOTEL (not date) so each worker keeps its proxy session
|
||||
alive across all dates for one hotel — looks like a single user checking
|
||||
availability for a trip.
|
||||
"""
|
||||
# Group by hotel to get stable shards
|
||||
hotels_seen: List[Dict[str, Any]] = []
|
||||
hotel_dates: Dict[int, List[date]] = {}
|
||||
for hotel, rate_date in hotel_date_jobs:
|
||||
hid = hotel['id']
|
||||
if hid not in hotel_dates:
|
||||
hotel_dates[hid] = []
|
||||
hotels_seen.append(hotel)
|
||||
hotel_dates[hid].append(rate_date)
|
||||
|
||||
# Shard hotels across workers
|
||||
shards: List[List[Dict[str, Any]]] = [hotels_seen[i::concurrency] for i in range(concurrency)]
|
||||
shards = [s for s in shards if s]
|
||||
|
||||
async def worker(hotel_shard: List[Dict[str, Any]], widx: int) -> Dict[str, int]:
|
||||
acc = {'rates': 0, 'completed': 0, 'failed': 0, 'blocked': 0}
|
||||
wdb = SyncSessionLocal()
|
||||
backend = PlaywrightHotelPageBackend(proxy_config=proxy_util.load_config(wdb))
|
||||
try:
|
||||
for hotel in hotel_shard:
|
||||
for rate_date in hotel_dates[hotel['id']]:
|
||||
try:
|
||||
result = await scrape_hotel_date(wdb, hotel, rate_date, backend, batch_id, adults)
|
||||
except Exception as e:
|
||||
logger.error(f"[worker {widx}] {hotel['name']} {rate_date} crashed: {e}")
|
||||
try:
|
||||
wdb.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
acc['failed'] += 1
|
||||
continue
|
||||
|
||||
if result.get('blocked'):
|
||||
acc['blocked'] += 1
|
||||
acc['failed'] += 1
|
||||
elif result['success']:
|
||||
acc['rates'] += result['rates_count']
|
||||
acc['completed'] += 1
|
||||
else:
|
||||
acc['failed'] += 1
|
||||
finally:
|
||||
try:
|
||||
await backend.close()
|
||||
except Exception:
|
||||
pass
|
||||
wdb.close()
|
||||
return acc
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(worker(shard, i) for i, shard in enumerate(shards)),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
agg = {'rates': 0, 'completed': 0, 'failed': 0, 'blocked': 0}
|
||||
for r in results:
|
||||
if isinstance(r, Exception):
|
||||
logger.error(f"Hotel-page scrape worker crashed: {r}")
|
||||
continue
|
||||
for k in agg:
|
||||
agg[k] += r[k]
|
||||
return agg
|
||||
|
||||
|
||||
def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
|
||||
"""mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
|
||||
if queue_id is None:
|
||||
|
|
@ -565,12 +723,105 @@ async def run_manual_scrape(
|
|||
_release_scrape_lock()
|
||||
|
||||
|
||||
async def _run_hotel_page_scrape(
|
||||
db: Session,
|
||||
from_date: date,
|
||||
to_date: date,
|
||||
scrape_type: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Hotel-page scrape mode: load each known hotel's property page per date.
|
||||
|
||||
Unlike search-results mode, this scrapes known hotels (from booking_com_hotels
|
||||
WHERE is_active AND booking_com_url IS NOT NULL) rather than discovering them
|
||||
from search results. Workers are sharded by hotel so each worker's proxy session
|
||||
covers all dates for one hotel before moving to the next.
|
||||
"""
|
||||
hotels = get_active_hotels(db)
|
||||
if not hotels:
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'No active hotels with booking_com_url in database. '
|
||||
'Run a search-results scrape first to populate hotels.',
|
||||
}
|
||||
|
||||
dates: List[date] = []
|
||||
current_date = from_date
|
||||
while current_date <= to_date:
|
||||
dates.append(current_date)
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
# Build (hotel, date) job pairs
|
||||
hotel_date_jobs: List[Tuple[Dict[str, Any], date]] = [
|
||||
(hotel, d) for hotel in hotels for d in dates
|
||||
]
|
||||
|
||||
batch_id = create_scrape_batch(db, scrape_type)
|
||||
db.execute(
|
||||
text("UPDATE booking_scrape_log SET dates_queued = :n WHERE batch_id = :bid"),
|
||||
{'n': len(hotel_date_jobs), 'bid': str(batch_id)}
|
||||
)
|
||||
db.commit()
|
||||
|
||||
concurrency = _effective_concurrency(db, len(hotels))
|
||||
logger.info(
|
||||
f"Hotel-page scrape {from_date}..{to_date}: "
|
||||
f"{len(hotels)} hotels × {len(dates)} dates = {len(hotel_date_jobs)} jobs, "
|
||||
f"{concurrency} worker(s)"
|
||||
)
|
||||
|
||||
try:
|
||||
config = get_scrape_config(db)
|
||||
adults = config['adults'] if config else 2
|
||||
agg = await _scrape_hotels_concurrent(hotel_date_jobs, concurrency, batch_id, adults)
|
||||
|
||||
status = 'completed' if agg['completed'] else 'failed'
|
||||
update_scrape_batch(
|
||||
db, batch_id,
|
||||
status=status,
|
||||
hotels_found=len(hotels),
|
||||
rates_scraped=agg['rates'],
|
||||
)
|
||||
db.execute(
|
||||
text("""
|
||||
UPDATE booking_scrape_log SET
|
||||
dates_completed = :completed, dates_failed = :failed
|
||||
WHERE batch_id = :bid
|
||||
"""),
|
||||
{'completed': agg['completed'], 'failed': agg['failed'], 'bid': str(batch_id)}
|
||||
)
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
'success': agg['completed'] > 0,
|
||||
'blocked': agg['blocked'] > 0,
|
||||
'dates_completed': agg['completed'],
|
||||
'dates_failed': agg['failed'],
|
||||
'hotels_found': len(hotels),
|
||||
'rates_scraped': agg['rates'],
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Hotel-page scrape error: {e}")
|
||||
update_scrape_batch(db, batch_id, status='failed', error_message=str(e))
|
||||
return {
|
||||
'success': False,
|
||||
'error': str(e),
|
||||
'dates_completed': 0,
|
||||
'dates_failed': len(hotel_date_jobs),
|
||||
}
|
||||
|
||||
|
||||
async def _run_manual_scrape_locked(
|
||||
db: Session,
|
||||
from_date: date,
|
||||
to_date: date
|
||||
) -> Dict[str, Any]:
|
||||
# Get config
|
||||
# Hotel-page mode: iterate known hotels × dates
|
||||
if _is_hotel_page_mode(db):
|
||||
return await _run_hotel_page_scrape(db, from_date, to_date, 'manual')
|
||||
|
||||
# Search-results mode (original)
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
return {
|
||||
|
|
@ -751,6 +1002,17 @@ async def process_queue(db: Session) -> Dict[str, Any]:
|
|||
|
||||
|
||||
async def _process_queue_locked(db: Session) -> Dict[str, Any]:
|
||||
# Hotel-page mode ignores the queue and just scrapes the queued date range directly
|
||||
if _is_hotel_page_mode(db):
|
||||
items = get_pending_queue_items(db, limit=200)
|
||||
if not items:
|
||||
return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'}
|
||||
dates = [it['rate_date'] for it in items]
|
||||
# Mark all as completed (hotel-page scrape manages its own tracking)
|
||||
for it in items:
|
||||
_safe_mark_queue(db, it['id'], 'completed')
|
||||
return await _run_hotel_page_scrape(db, min(dates), max(dates), 'scheduled')
|
||||
|
||||
# Get config
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ Provides pluggable backends to allow switching between:
|
|||
|
||||
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
|
||||
from .playwright_local import PlaywrightLocalBackend
|
||||
from .playwright_hotel_page import PlaywrightHotelPageBackend
|
||||
|
||||
__all__ = [
|
||||
'ScraperBackend',
|
||||
|
|
@ -17,4 +18,5 @@ __all__ = [
|
|||
'RateData',
|
||||
'AvailabilityStatus',
|
||||
'PlaywrightLocalBackend',
|
||||
'PlaywrightHotelPageBackend',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ class RateData:
|
|||
no_prepayment: Optional[bool] = None
|
||||
rooms_left: Optional[int] = None # "Only X rooms left"
|
||||
available_qty: Optional[int] = None # Future: from hotel page dropdown
|
||||
rate_plan_id: Optional[str] = None # block_id from hotel page (identifies rate variant)
|
||||
max_persons: Optional[int] = None # Occupancy this rate applies to
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
328
backend/services/scraper_backends/playwright_hotel_page.py
Normal file
328
backend/services/scraper_backends/playwright_hotel_page.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
"""
|
||||
Booking.com Hotel Page Scraper Backend
|
||||
|
||||
Scrapes individual hotel property pages instead of search results.
|
||||
Returns all room types and rate plan variants per hotel per date.
|
||||
|
||||
Advantages over search-results approach:
|
||||
- Individual hotel pages are not Cloudflare-protected (search results are)
|
||||
- Captures every room type, rate plan, meal plan, and availability count
|
||||
- Proxy reuse: one residential IP stays valid across many hotel pages
|
||||
(looks like a human browsing properties), rotates only when blocked
|
||||
|
||||
Rate plan variants per room type (typical):
|
||||
2-adult + room only + non-refundable
|
||||
2-adult + room only + free cancellation
|
||||
2-adult + breakfast + non-refundable
|
||||
2-adult + breakfast + free cancellation
|
||||
1-adult + room only / breakfast (filtered out by caller if not needed)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from typing import List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
||||
|
||||
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# JS that extracts all rate plan rows from the room availability table.
|
||||
# Runs inside the page after the room table has loaded.
|
||||
_EXTRACT_RATES_JS = """
|
||||
() => {
|
||||
const results = [];
|
||||
|
||||
document.querySelectorAll('[id^="room_type_id_"]').forEach(roomEl => {
|
||||
const roomId = roomEl.getAttribute('data-room-id') || roomEl.id.replace('room_type_id_', '');
|
||||
const roomName = (
|
||||
roomEl.querySelector('.hprt-roomtype-icon-link')?.innerText ||
|
||||
roomEl.querySelector('span')?.innerText || ''
|
||||
).trim();
|
||||
|
||||
// Availability count from scarcity indicator ("We have 2 left")
|
||||
const availText = roomEl.closest('tr')
|
||||
?.querySelector('.only_x_left, .thisRoomAvailabilityNew span')
|
||||
?.innerText?.trim() || '';
|
||||
const availMatch = availText.match(/\\d+/);
|
||||
const availCount = availMatch ? parseInt(availMatch[0]) : null;
|
||||
|
||||
// Walk sibling <tr> rows that belong to this room type
|
||||
let tr = roomEl.closest('tr');
|
||||
while (tr) {
|
||||
if (tr.classList.contains('js-rt-block-row')) {
|
||||
const blockId = tr.getAttribute('data-block-id') || '';
|
||||
const priceRaw = tr.getAttribute('data-hotel-rounded-price') || '';
|
||||
|
||||
let fltrs = {};
|
||||
try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {}
|
||||
|
||||
// Conditions cell (3rd <td>) holds meal plan + cancel info
|
||||
const cells = tr.querySelectorAll('td');
|
||||
const condCell = cells.length >= 3 ? cells[2].innerText || '' : '';
|
||||
|
||||
const breakfastIncluded = condCell.toLowerCase().includes('breakfast');
|
||||
const nonRefundable = (fltrs.non_refundable === 1);
|
||||
|
||||
// Free cancellation date: "Free cancellation before DD Month YYYY"
|
||||
const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
|
||||
const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
|
||||
|
||||
// Persons: first <td> ("Max persons: 2" or "Only for 1 guest")
|
||||
const personsCell = cells.length >= 1 ? cells[0].innerText || '' : '';
|
||||
const personsMatch = personsCell.match(/\\d+/);
|
||||
const maxPersons = personsMatch ? parseInt(personsMatch[0]) : null;
|
||||
|
||||
results.push({
|
||||
room_id: roomId,
|
||||
room_name: roomName,
|
||||
avail_count: availCount,
|
||||
block_id: blockId,
|
||||
price: priceRaw ? parseInt(priceRaw) : null,
|
||||
breakfast_included: breakfastIncluded,
|
||||
non_refundable: nonRefundable,
|
||||
free_cancel_text: freeCancelText,
|
||||
max_persons: maxPersons,
|
||||
});
|
||||
}
|
||||
|
||||
tr = tr.nextElementSibling;
|
||||
if (!tr) break;
|
||||
// Stop at the next room type's header row
|
||||
if (tr.querySelector('[id^="room_type_id_"]')) break;
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _slug_from_url(url: str) -> Optional[str]:
|
||||
"""Extract hotel slug from Booking.com hotel page URL."""
|
||||
if not url:
|
||||
return None
|
||||
m = re.search(r'/hotel/\w+/([^.]+)\.', url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _build_hotel_url(hotel_url: str, checkin: date, checkout: date, adults: int) -> str:
|
||||
base = hotel_url.split('?')[0]
|
||||
return (
|
||||
f"{base}?checkin={checkin}&checkout={checkout}"
|
||||
f"&group_adults={adults}&no_rooms=1&selected_currency=GBP"
|
||||
)
|
||||
|
||||
|
||||
class PlaywrightHotelPageBackend(ScraperBackend):
|
||||
"""
|
||||
Scrapes individual Booking.com hotel property pages.
|
||||
|
||||
Keeps one browser context alive across multiple hotel+date requests.
|
||||
Rotates proxy only when a CF/WAF block is detected.
|
||||
"""
|
||||
|
||||
def __init__(self, proxy_config: dict = None):
|
||||
self._proxy_config = proxy_config or {}
|
||||
self._pw = None
|
||||
self._browser: Optional[Browser] = None
|
||||
self._context: Optional[BrowserContext] = None
|
||||
self._requests_on_context = 0
|
||||
# Rotate after this many requests even without a block (keeps session fresh)
|
||||
self._max_requests_per_context = 40
|
||||
|
||||
def _proxy_enabled(self) -> bool:
|
||||
return bool(self._proxy_config.get('server'))
|
||||
|
||||
def _proxy_kwargs(self) -> dict:
|
||||
if not self._proxy_enabled():
|
||||
return {}
|
||||
cfg = self._proxy_config
|
||||
proxy = {'server': cfg['server']}
|
||||
if cfg.get('username'):
|
||||
proxy['username'] = cfg['username']
|
||||
if cfg.get('password'):
|
||||
proxy['password'] = cfg['password']
|
||||
return {'proxy': proxy}
|
||||
|
||||
async def _start(self):
|
||||
if not self._pw:
|
||||
self._pw = await async_playwright().start()
|
||||
if not self._browser:
|
||||
self._browser = await self._pw.chromium.launch(
|
||||
headless=True,
|
||||
args=['--no-sandbox', '--disable-dev-shm-usage'],
|
||||
)
|
||||
|
||||
async def _get_context(self) -> BrowserContext:
|
||||
await self._start()
|
||||
if self._context is None or self._requests_on_context >= self._max_requests_per_context:
|
||||
if self._context:
|
||||
try:
|
||||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._context = await self._browser.new_context(
|
||||
user_agent=(
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
'Chrome/125.0.0.0 Safari/537.36'
|
||||
),
|
||||
**self._proxy_kwargs(),
|
||||
)
|
||||
self._requests_on_context = 0
|
||||
logger.debug("Opened new browser context" + (" (with proxy)" if self._proxy_enabled() else ""))
|
||||
return self._context
|
||||
|
||||
async def _rotate_context(self):
|
||||
"""Force a new browser context (new proxy session)."""
|
||||
if self._context:
|
||||
try:
|
||||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._context = None
|
||||
self._requests_on_context = 0
|
||||
logger.info("Rotated browser context (new proxy session)")
|
||||
|
||||
async def scrape_hotel_page(
|
||||
self,
|
||||
hotel_url: str,
|
||||
check_in: date,
|
||||
check_out: date,
|
||||
adults: int = 2,
|
||||
) -> ScraperResult:
|
||||
"""
|
||||
Load a hotel page and extract all room types + rate plan variants.
|
||||
|
||||
On block detection, rotates proxy and returns blocked=True so the
|
||||
caller can retry with the fresh context.
|
||||
"""
|
||||
page_url = _build_hotel_url(hotel_url, check_in, check_out, adults)
|
||||
context = await self._get_context()
|
||||
self._requests_on_context += 1
|
||||
page: Optional[Page] = None
|
||||
|
||||
try:
|
||||
page = await context.new_page()
|
||||
|
||||
# Load the page
|
||||
page_loaded = False
|
||||
try:
|
||||
await page.goto(page_url, wait_until='domcontentloaded', timeout=30000)
|
||||
page_loaded = True
|
||||
except Exception as e:
|
||||
logger.warning(f"goto timeout for {hotel_url} {check_in}: {e}")
|
||||
|
||||
if not page_loaded:
|
||||
return ScraperResult(success=False, blocked=False, error_message='page load timeout')
|
||||
|
||||
# Wait for the room table (prices render via JS after DOM)
|
||||
room_table_appeared = False
|
||||
try:
|
||||
await page.wait_for_selector('[id^="room_type_id_"]', timeout=15000)
|
||||
room_table_appeared = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not room_table_appeared:
|
||||
# Could be sold-out, no-availability, or a block page
|
||||
content = await page.content()
|
||||
is_blocked, reason = self.detect_blocking(content)
|
||||
if is_blocked or len(content) < 20000:
|
||||
logger.warning(f"Block detected for {hotel_url} {check_in}: {reason or 'small page'}")
|
||||
await self._rotate_context()
|
||||
return ScraperResult(success=False, blocked=True, block_reason=reason or 'small page')
|
||||
|
||||
# No availability for this date
|
||||
return ScraperResult(
|
||||
success=True,
|
||||
rates=[RateData(
|
||||
rate_date=check_in,
|
||||
availability_status=AvailabilityStatus.SOLD_OUT,
|
||||
currency='GBP',
|
||||
)],
|
||||
)
|
||||
|
||||
# Extra wait for JS prices to render
|
||||
await asyncio.sleep(10)
|
||||
|
||||
# Check for explicit "no availability" message
|
||||
no_avail_el = await page.query_selector('#no_availability_msg')
|
||||
if no_avail_el and await no_avail_el.is_visible():
|
||||
return ScraperResult(
|
||||
success=True,
|
||||
rates=[RateData(
|
||||
rate_date=check_in,
|
||||
availability_status=AvailabilityStatus.SOLD_OUT,
|
||||
currency='GBP',
|
||||
)],
|
||||
)
|
||||
|
||||
# Extract all rate plan rows
|
||||
raw_plans = await page.evaluate(_EXTRACT_RATES_JS)
|
||||
|
||||
if not raw_plans:
|
||||
return ScraperResult(success=True, rates=[])
|
||||
|
||||
rates: List[RateData] = []
|
||||
for plan in raw_plans:
|
||||
if plan['price'] is None:
|
||||
continue
|
||||
rates.append(RateData(
|
||||
rate_date=check_in,
|
||||
availability_status=AvailabilityStatus.AVAILABLE,
|
||||
rate_gross=Decimal(plan['price']),
|
||||
currency='GBP',
|
||||
room_type=plan['room_name'] or None,
|
||||
breakfast_included=plan['breakfast_included'],
|
||||
free_cancellation=not plan['non_refundable'],
|
||||
rooms_left=plan['avail_count'],
|
||||
rate_plan_id=plan['block_id'] or None,
|
||||
max_persons=plan['max_persons'],
|
||||
))
|
||||
|
||||
logger.info(
|
||||
f"Hotel page {_slug_from_url(hotel_url)} {check_in}: "
|
||||
f"{len(raw_plans)} rate plans extracted"
|
||||
)
|
||||
return ScraperResult(success=True, rates=rates)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"scrape_hotel_page error for {hotel_url} {check_in}: {e}")
|
||||
return ScraperResult(success=False, error_message=str(e))
|
||||
finally:
|
||||
if page:
|
||||
try:
|
||||
await page.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def scrape_location_search(self, *args, **kwargs) -> ScraperResult:
|
||||
"""Not used by this backend — hotel pages replace location search."""
|
||||
raise NotImplementedError("PlaywrightHotelPageBackend does not support location search")
|
||||
|
||||
async def close(self):
|
||||
if self._context:
|
||||
try:
|
||||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._context = None
|
||||
if self._browser:
|
||||
try:
|
||||
await self._browser.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._browser = None
|
||||
if self._pw:
|
||||
try:
|
||||
await self._pw.stop()
|
||||
except Exception:
|
||||
pass
|
||||
self._pw = None
|
||||
Loading…
Add table
Add a link
Reference in a new issue