Pin Booking.com search to a dest_id
Free-text ss= destination resolution is non-deterministic: tonight's 30-day run resolved "Stow on the Wold" to St. Wolfgang, Austria for 8 of 30 dates, saving Salzkammergut hotel rates into the matrix. dest_id + dest_type=city in the search URL pins the destination. - booking_scrape_config gains a dest_id column (idempotent ALTER) - scrape_location_search/_build_search_url thread dest_id through - /config/location accepts dest_id Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
270d8293d1
commit
9e5728efb1
5 changed files with 34 additions and 12 deletions
|
|
@ -90,7 +90,8 @@ class ScraperBackend(ABC):
|
|||
check_in: date,
|
||||
check_out: date,
|
||||
adults: int = 2,
|
||||
pages: int = 2
|
||||
pages: int = 2,
|
||||
dest_id: Optional[str] = None
|
||||
) -> ScraperResult:
|
||||
"""
|
||||
Scrape booking.com location search results.
|
||||
|
|
@ -101,6 +102,9 @@ class ScraperBackend(ABC):
|
|||
check_out: Check-out date (typically check_in + 1 for single night)
|
||||
adults: Number of adults for search
|
||||
pages: Number of search result pages to scrape
|
||||
dest_id: Booking.com numeric destination id. Pins the search to one
|
||||
destination — free-text ss= resolves non-deterministically
|
||||
(Stow on the Wold intermittently matched St. Wolfgang, Austria)
|
||||
|
||||
Returns:
|
||||
ScraperResult with hotels and rates found
|
||||
|
|
|
|||
|
|
@ -276,7 +276,8 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
check_in: date,
|
||||
check_out: date,
|
||||
adults: int,
|
||||
offset: int = 0
|
||||
offset: int = 0,
|
||||
dest_id: Optional[str] = None
|
||||
) -> str:
|
||||
"""Build booking.com search URL with parameters."""
|
||||
params = {
|
||||
|
|
@ -287,6 +288,11 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
'no_rooms': 1,
|
||||
'group_children': 0,
|
||||
}
|
||||
if dest_id:
|
||||
# Pin the destination — without this, free-text ss= intermittently
|
||||
# resolves to the wrong place entirely.
|
||||
params['dest_id'] = dest_id
|
||||
params['dest_type'] = 'city'
|
||||
if offset > 0:
|
||||
params['offset'] = offset
|
||||
|
||||
|
|
@ -502,7 +508,8 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
check_in: date,
|
||||
check_out: date,
|
||||
adults: int = 2,
|
||||
pages: int = 2
|
||||
pages: int = 2,
|
||||
dest_id: Optional[str] = None
|
||||
) -> ScraperResult:
|
||||
"""
|
||||
Scrape booking.com location search results, rotating the proxy IP if
|
||||
|
|
@ -515,6 +522,7 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
check_out: Check-out date (check_in + 1 for single night rate)
|
||||
adults: Number of adults
|
||||
pages: Number of result pages to scrape
|
||||
dest_id: Booking.com numeric destination id (pins the search)
|
||||
|
||||
Returns:
|
||||
ScraperResult with hotels and rates found
|
||||
|
|
@ -523,7 +531,7 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
result = None
|
||||
|
||||
for attempt in range(max_attempts):
|
||||
result = await self._scrape_once(location, check_in, check_out, adults, pages)
|
||||
result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id)
|
||||
|
||||
# Soft-block signals: an explicit challenge, or a "successful" load
|
||||
# that yielded zero hotels (page 1 never rendered results).
|
||||
|
|
@ -547,7 +555,8 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
check_in: date,
|
||||
check_out: date,
|
||||
adults: int,
|
||||
pages: int
|
||||
pages: int,
|
||||
dest_id: Optional[str] = None
|
||||
) -> ScraperResult:
|
||||
"""A single scrape attempt for one date on the current IP/session."""
|
||||
all_hotels = []
|
||||
|
|
@ -574,7 +583,7 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
page_loaded = True
|
||||
if page_num == 0:
|
||||
# Navigate to page 1 by URL
|
||||
url = self._build_search_url(location, check_in, check_out, adults)
|
||||
url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id)
|
||||
logger.info(f"Scraping page {page_num + 1}: {url}")
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
|
|
@ -588,7 +597,8 @@ class PlaywrightLocalBackend(ScraperBackend):
|
|||
if not clicked:
|
||||
# Fall back to offset URL if the control isn't found
|
||||
url = self._build_search_url(
|
||||
location, check_in, check_out, adults, offset=page_num * 25
|
||||
location, check_in, check_out, adults,
|
||||
offset=page_num * 25, dest_id=dest_id
|
||||
)
|
||||
logger.info(f"Next-button not found, offset fallback: {url}")
|
||||
try:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue