hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Pin Booking.com search to a dest_id

Browse source 
Free-text ss= destination resolution is non-deterministic: tonight's
30-day run resolved "Stow on the Wold" to St. Wolfgang, Austria for 8 of
30 dates, saving Salzkammergut hotel rates into the matrix. dest_id +
dest_type=city in the search URL pins the destination.

- booking_scrape_config gains a dest_id column (idempotent ALTER)
- scrape_location_search/_build_search_url thread dest_id through
- /config/location accepts dest_id

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 21:49:57 +00:00
parent 270d8293d1
commit 9e5728efb1
5 changed files with 34 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

8
backend/api/competitors.py
View file

@ -30,6 +30,7 @@ class LocationConfigRequest(BaseModel):
location_name: str
pages_to_scrape: int = 2
adults: int = 2
dest_id: Optional[str] = None # Booking.com numeric destination id (pins the search)
class HotelTierUpdate(BaseModel):
@ -310,10 +311,11 @@ async def set_location_config(
# Insert new config
await db.execute(
text("""
INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, is_active)
VALUES (:location, :pages, :adults, TRUE)
INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, dest_id, is_active)
VALUES (:location, :pages, :adults, :dest_id, TRUE)
"""),
{'location': config.location_name, 'pages': config.pages_to_scrape, 'adults': config.adults}
{'location': config.location_name, 'pages': config.pages_to_scrape,
'adults': config.adults, 'dest_id': config.dest_id}
)
await db.commit()

4
backend/schema.sql
View file

@ -279,3 +279,7 @@ CREATE INDEX IF NOT EXISTS idx_direct_rates_scraped ON direct_rates(scraped_a
-- Link booking_com_hotels to direct_competitor_hotels (optional, for Market View direct column)
ALTER TABLE booking_com_hotels ADD COLUMN IF NOT EXISTS direct_hotel_id INTEGER
REFERENCES direct_competitor_hotels(id) ON DELETE SET NULL;
-- Pin the Booking.com destination: free-text ss= searches non-deterministically
-- resolve to the wrong place (Stow on the Wold once matched St. Wolfgang, AT).
ALTER TABLE booking_scrape_config ADD COLUMN IF NOT EXISTS dest_id VARCHAR(32);

6
backend/services/booking_scraper.py
View file

@ -82,7 +82,7 @@ def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
"""Get the active scrape location configuration."""
result = db.execute(
text("""
SELECT id, location_name, location_search_url, pages_to_scrape, adults
SELECT id, location_name, location_search_url, pages_to_scrape, adults, dest_id
FROM booking_scrape_config
WHERE is_active = TRUE
ORDER BY id
@ -99,6 +99,7 @@ def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
'location_search_url': result.location_search_url,
'pages_to_scrape': result.pages_to_scrape or 2,
'adults': result.adults or 2,
'dest_id': result.dest_id,
}
@ -330,7 +331,8 @@ async def scrape_date(
check_in=check_in,
check_out=check_out,
adults=config['adults'],
pages=config['pages_to_scrape']
pages=config['pages_to_scrape'],
dest_id=config.get('dest_id'),
)
if result.blocked:

6
backend/services/scraper_backends/base.py
View file

@ -90,7 +90,8 @@ class ScraperBackend(ABC):
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
pages: int = 2,
dest_id: Optional[str] = None
) -> ScraperResult:
"""
Scrape booking.com location search results.
@ -101,6 +102,9 @@ class ScraperBackend(ABC):
check_out: Check-out date (typically check_in + 1 for single night)
adults: Number of adults for search
pages: Number of search result pages to scrape
dest_id: Booking.com numeric destination id. Pins the search to one
destination — free-text ss= resolves non-deterministically
(Stow on the Wold intermittently matched St. Wolfgang, Austria)
Returns:
ScraperResult with hotels and rates found

22
backend/services/scraper_backends/playwright_local.py
View file

@ -276,7 +276,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_in: date,
check_out: date,
adults: int,
offset: int = 0
offset: int = 0,
dest_id: Optional[str] = None
) -> str:
"""Build booking.com search URL with parameters."""
params = {
@ -287,6 +288,11 @@ class PlaywrightLocalBackend(ScraperBackend):
'no_rooms': 1,
'group_children': 0,
}
if dest_id:
# Pin the destination — without this, free-text ss= intermittently
# resolves to the wrong place entirely.
params['dest_id'] = dest_id
params['dest_type'] = 'city'
if offset > 0:
params['offset'] = offset
@ -502,7 +508,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
pages: int = 2,
dest_id: Optional[str] = None
) -> ScraperResult:
"""
Scrape booking.com location search results, rotating the proxy IP if
@ -515,6 +522,7 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: Check-out date (check_in + 1 for single night rate)
adults: Number of adults
pages: Number of result pages to scrape
dest_id: Booking.com numeric destination id (pins the search)
Returns:
ScraperResult with hotels and rates found
@ -523,7 +531,7 @@ class PlaywrightLocalBackend(ScraperBackend):
result = None
for attempt in range(max_attempts):
result = await self._scrape_once(location, check_in, check_out, adults, pages)
result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id)
# Soft-block signals: an explicit challenge, or a "successful" load
# that yielded zero hotels (page 1 never rendered results).
@ -547,7 +555,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_in: date,
check_out: date,
adults: int,
pages: int
pages: int,
dest_id: Optional[str] = None
) -> ScraperResult:
"""A single scrape attempt for one date on the current IP/session."""
all_hotels = []
@ -574,7 +583,7 @@ class PlaywrightLocalBackend(ScraperBackend):
page_loaded = True
if page_num == 0:
# Navigate to page 1 by URL
url = self._build_search_url(location, check_in, check_out, adults)
url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
@ -588,7 +597,8 @@ class PlaywrightLocalBackend(ScraperBackend):
if not clicked:
# Fall back to offset URL if the control isn't found
url = self._build_search_url(
location, check_in, check_out, adults, offset=page_num * 25
location, check_in, check_out, adults,
offset=page_num * 25, dest_id=dest_id
)
logger.info(f"Next-button not found, offset fallback: {url}")
try: