hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Use pasted Booking.com search URL to pin scrape destination

Browse source 
The location_search_url column existed but was dead — the scraper always
rebuilt the URL from the free-text location name. Now the Location
Configuration form takes a "Booking.com search URL" field: paste the
address-bar URL from a real search and the scraper lifts ss/dest_id/
dest_type from it (the most reliable destination pin). Falls back to
dest_id, then plain name. Server also extracts dest_id from the URL for
the column and derives a display name from ss when none is typed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 22:00:28 +00:00
parent 9e5728efb1
commit 579a189cb8
5 changed files with 97 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

5
backend/services/scraper_backends/base.py
View file

@ -91,7 +91,8 @@ class ScraperBackend(ABC):
check_out: date,
adults: int = 2,
pages: int = 2,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
search_url: Optional[str] = None
) -> ScraperResult:
"""
Scrape booking.com location search results.
@ -105,6 +106,8 @@ class ScraperBackend(ABC):
dest_id: Booking.com numeric destination id. Pins the search to one
destination — free-text ss= resolves non-deterministically
(Stow on the Wold intermittently matched St. Wolfgang, Austria)
search_url: A pasted Booking.com search URL whose ss/dest_id/
dest_type pin the destination (wins over dest_id/location)
Returns:
ScraperResult with hotels and rates found

62
backend/services/scraper_backends/playwright_local.py
View file

@ -13,7 +13,24 @@ import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode, urlparse
from urllib.parse import urlencode, urlparse, parse_qs
def location_params_from_url(url: str) -> dict:
"""Pull the stable destination params (ss, dest_id, dest_type) out of a
pasted Booking.com search URL, ignoring volatile session/click params and
the date/occupancy params we set ourselves per scrape."""
if not url:
return {}
try:
q = parse_qs(urlparse(url).query)
except Exception:
return {}
out = {}
for key in ('ss', 'dest_id', 'dest_type'):
if q.get(key):
out[key] = q[key][0]
return out
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
@ -277,22 +294,30 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: date,
adults: int,
offset: int = 0,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
location_params: Optional[dict] = None
) -> str:
"""Build booking.com search URL with parameters."""
"""Build booking.com search URL. We always own the date/occupancy/paging
params; the destination comes from (in priority order) a pasted search
URL's params, an explicit dest_id, or the free-text location name."""
params = {
'ss': location,
'checkin': check_in.isoformat(),
'checkout': check_out.isoformat(),
'group_adults': adults,
'no_rooms': 1,
'group_children': 0,
}
if dest_id:
# Pin the destination — without this, free-text ss= intermittently
# resolves to the wrong place entirely.
params['dest_id'] = dest_id
params['dest_type'] = 'city'
if location_params:
# ss + dest_id + dest_type lifted from the user's pasted URL —
# the most reliable pin (it's exactly what their browser resolved).
params.update(location_params)
else:
params['ss'] = location
if dest_id:
# Pin the destination — without this, free-text ss= intermittently
# resolves to the wrong place entirely.
params['dest_id'] = dest_id
params['dest_type'] = 'city'
if offset > 0:
params['offset'] = offset
@ -509,7 +534,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: date,
adults: int = 2,
pages: int = 2,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
search_url: Optional[str] = None
) -> ScraperResult:
"""
Scrape booking.com location search results, rotating the proxy IP if
@ -523,15 +549,19 @@ class PlaywrightLocalBackend(ScraperBackend):
adults: Number of adults
pages: Number of result pages to scrape
dest_id: Booking.com numeric destination id (pins the search)
search_url: A pasted Booking.com search URL — its ss/dest_id/
dest_type win over dest_id/location when present
Returns:
ScraperResult with hotels and rates found
"""
location_params = location_params_from_url(search_url) if search_url else None
max_attempts = 3 if self._proxy_enabled() else 1
result = None
for attempt in range(max_attempts):
result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id)
result = await self._scrape_once(
location, check_in, check_out, adults, pages, dest_id, location_params)
# Soft-block signals: an explicit challenge, or a "successful" load
# that yielded zero hotels (page 1 never rendered results).
@ -556,7 +586,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: date,
adults: int,
pages: int,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
location_params: Optional[dict] = None
) -> ScraperResult:
"""A single scrape attempt for one date on the current IP/session."""
all_hotels = []
@ -583,7 +614,9 @@ class PlaywrightLocalBackend(ScraperBackend):
page_loaded = True
if page_num == 0:
# Navigate to page 1 by URL
url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id)
url = self._build_search_url(
location, check_in, check_out, adults,
dest_id=dest_id, location_params=location_params)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
@ -598,7 +631,8 @@ class PlaywrightLocalBackend(ScraperBackend):
# Fall back to offset URL if the control isn't found
url = self._build_search_url(
location, check_in, check_out, adults,
offset=page_num * 25, dest_id=dest_id
offset=page_num * 25, dest_id=dest_id,
location_params=location_params
)
logger.info(f"Next-button not found, offset fallback: {url}")
try: