diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 420b2f6..13cdc8c 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -30,6 +30,7 @@ class LocationConfigRequest(BaseModel): location_name: str pages_to_scrape: int = 2 adults: int = 2 + dest_id: Optional[str] = None # Booking.com numeric destination id (pins the search) class HotelTierUpdate(BaseModel): @@ -310,10 +311,11 @@ async def set_location_config( # Insert new config await db.execute( text(""" - INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, is_active) - VALUES (:location, :pages, :adults, TRUE) + INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, dest_id, is_active) + VALUES (:location, :pages, :adults, :dest_id, TRUE) """), - {'location': config.location_name, 'pages': config.pages_to_scrape, 'adults': config.adults} + {'location': config.location_name, 'pages': config.pages_to_scrape, + 'adults': config.adults, 'dest_id': config.dest_id} ) await db.commit() diff --git a/backend/schema.sql b/backend/schema.sql index 85564e6..b85c902 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -279,3 +279,7 @@ CREATE INDEX IF NOT EXISTS idx_direct_rates_scraped ON direct_rates(scraped_a -- Link booking_com_hotels to direct_competitor_hotels (optional, for Market View direct column) ALTER TABLE booking_com_hotels ADD COLUMN IF NOT EXISTS direct_hotel_id INTEGER REFERENCES direct_competitor_hotels(id) ON DELETE SET NULL; + +-- Pin the Booking.com destination: free-text ss= searches non-deterministically +-- resolve to the wrong place (Stow on the Wold once matched St. Wolfgang, AT). +ALTER TABLE booking_scrape_config ADD COLUMN IF NOT EXISTS dest_id VARCHAR(32); diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 826d897..6fe22cc 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -82,7 +82,7 @@ def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: """Get the active scrape location configuration.""" result = db.execute( text(""" - SELECT id, location_name, location_search_url, pages_to_scrape, adults + SELECT id, location_name, location_search_url, pages_to_scrape, adults, dest_id FROM booking_scrape_config WHERE is_active = TRUE ORDER BY id @@ -99,6 +99,7 @@ def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: 'location_search_url': result.location_search_url, 'pages_to_scrape': result.pages_to_scrape or 2, 'adults': result.adults or 2, + 'dest_id': result.dest_id, } @@ -330,7 +331,8 @@ async def scrape_date( check_in=check_in, check_out=check_out, adults=config['adults'], - pages=config['pages_to_scrape'] + pages=config['pages_to_scrape'], + dest_id=config.get('dest_id'), ) if result.blocked: diff --git a/backend/services/scraper_backends/base.py b/backend/services/scraper_backends/base.py index 271a74c..d8c34da 100644 --- a/backend/services/scraper_backends/base.py +++ b/backend/services/scraper_backends/base.py @@ -90,7 +90,8 @@ class ScraperBackend(ABC): check_in: date, check_out: date, adults: int = 2, - pages: int = 2 + pages: int = 2, + dest_id: Optional[str] = None ) -> ScraperResult: """ Scrape booking.com location search results. @@ -101,6 +102,9 @@ class ScraperBackend(ABC): check_out: Check-out date (typically check_in + 1 for single night) adults: Number of adults for search pages: Number of search result pages to scrape + dest_id: Booking.com numeric destination id. Pins the search to one + destination — free-text ss= resolves non-deterministically + (Stow on the Wold intermittently matched St. Wolfgang, Austria) Returns: ScraperResult with hotels and rates found diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 7802843..fc5d1cc 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -276,7 +276,8 @@ class PlaywrightLocalBackend(ScraperBackend): check_in: date, check_out: date, adults: int, - offset: int = 0 + offset: int = 0, + dest_id: Optional[str] = None ) -> str: """Build booking.com search URL with parameters.""" params = { @@ -287,6 +288,11 @@ class PlaywrightLocalBackend(ScraperBackend): 'no_rooms': 1, 'group_children': 0, } + if dest_id: + # Pin the destination — without this, free-text ss= intermittently + # resolves to the wrong place entirely. + params['dest_id'] = dest_id + params['dest_type'] = 'city' if offset > 0: params['offset'] = offset @@ -502,7 +508,8 @@ class PlaywrightLocalBackend(ScraperBackend): check_in: date, check_out: date, adults: int = 2, - pages: int = 2 + pages: int = 2, + dest_id: Optional[str] = None ) -> ScraperResult: """ Scrape booking.com location search results, rotating the proxy IP if @@ -515,6 +522,7 @@ class PlaywrightLocalBackend(ScraperBackend): check_out: Check-out date (check_in + 1 for single night rate) adults: Number of adults pages: Number of result pages to scrape + dest_id: Booking.com numeric destination id (pins the search) Returns: ScraperResult with hotels and rates found @@ -523,7 +531,7 @@ class PlaywrightLocalBackend(ScraperBackend): result = None for attempt in range(max_attempts): - result = await self._scrape_once(location, check_in, check_out, adults, pages) + result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id) # Soft-block signals: an explicit challenge, or a "successful" load # that yielded zero hotels (page 1 never rendered results). @@ -547,7 +555,8 @@ class PlaywrightLocalBackend(ScraperBackend): check_in: date, check_out: date, adults: int, - pages: int + pages: int, + dest_id: Optional[str] = None ) -> ScraperResult: """A single scrape attempt for one date on the current IP/session.""" all_hotels = [] @@ -574,7 +583,7 @@ class PlaywrightLocalBackend(ScraperBackend): page_loaded = True if page_num == 0: # Navigate to page 1 by URL - url = self._build_search_url(location, check_in, check_out, adults) + url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id) logger.info(f"Scraping page {page_num + 1}: {url}") try: await page.goto(url, wait_until='domcontentloaded', timeout=30000) @@ -588,7 +597,8 @@ class PlaywrightLocalBackend(ScraperBackend): if not clicked: # Fall back to offset URL if the control isn't found url = self._build_search_url( - location, check_in, check_out, adults, offset=page_num * 25 + location, check_in, check_out, adults, + offset=page_num * 25, dest_id=dest_id ) logger.info(f"Next-button not found, offset fallback: {url}") try: