From 579a189cb8ee3dbf4f76ea45da8930dc00e3ebeb Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Sun, 5 Jul 2026 22:00:28 +0000 Subject: [PATCH] Use pasted Booking.com search URL to pin scrape destination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The location_search_url column existed but was dead — the scraper always rebuilt the URL from the free-text location name. Now the Location Configuration form takes a "Booking.com search URL" field: paste the address-bar URL from a real search and the scraper lifts ss/dest_id/ dest_type from it (the most reliable destination pin). Falls back to dest_id, then plain name. Server also extracts dest_id from the URL for the column and derives a display name from ss when none is typed. Co-Authored-By: Claude Fable 5 --- backend/api/competitors.py | 27 ++++++-- backend/services/booking_scraper.py | 1 + backend/services/scraper_backends/base.py | 5 +- .../scraper_backends/playwright_local.py | 62 ++++++++++++++----- frontend/src/pages/MarketView.tsx | 26 +++++++- 5 files changed, 97 insertions(+), 24 deletions(-) diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 13cdc8c..3123e30 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -27,10 +27,11 @@ class ScrapeRequest(BaseModel): class LocationConfigRequest(BaseModel): - location_name: str + location_name: str = '' pages_to_scrape: int = 2 adults: int = 2 dest_id: Optional[str] = None # Booking.com numeric destination id (pins the search) + location_search_url: Optional[str] = None # Pasted Booking.com search URL (pins destination) class HotelTierUpdate(BaseModel): @@ -303,6 +304,19 @@ async def set_location_config( current_user: dict = Depends(get_current_user) ): """Set the location to scrape for competitor rates.""" + from services.scraper_backends.playwright_local import location_params_from_url + + # If a full search URL was pasted, lift the destination params out of it — + # dest_id for the column (visibility/fallback), and ss for the display name + # when the user didn't type one. + search_url = (config.location_search_url or '').strip() or None + url_params = location_params_from_url(search_url) if search_url else {} + dest_id = config.dest_id or url_params.get('dest_id') + location_name = config.location_name.strip() or url_params.get('ss', '') + + if not location_name: + raise HTTPException(status_code=400, detail="Provide a location name or a search URL.") + # Deactivate existing configs await db.execute( text("UPDATE booking_scrape_config SET is_active = FALSE") @@ -311,15 +325,16 @@ async def set_location_config( # Insert new config await db.execute( text(""" - INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, dest_id, is_active) - VALUES (:location, :pages, :adults, :dest_id, TRUE) + INSERT INTO booking_scrape_config + (location_name, pages_to_scrape, adults, dest_id, location_search_url, is_active) + VALUES (:location, :pages, :adults, :dest_id, :search_url, TRUE) """), - {'location': config.location_name, 'pages': config.pages_to_scrape, - 'adults': config.adults, 'dest_id': config.dest_id} + {'location': location_name, 'pages': config.pages_to_scrape, + 'adults': config.adults, 'dest_id': dest_id, 'search_url': search_url} ) await db.commit() - return {"status": "success", "location": config.location_name} + return {"status": "success", "location": location_name} @router.post("/config/enable") diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 6fe22cc..c3d400a 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -333,6 +333,7 @@ async def scrape_date( adults=config['adults'], pages=config['pages_to_scrape'], dest_id=config.get('dest_id'), + search_url=config.get('location_search_url'), ) if result.blocked: diff --git a/backend/services/scraper_backends/base.py b/backend/services/scraper_backends/base.py index d8c34da..92d74f3 100644 --- a/backend/services/scraper_backends/base.py +++ b/backend/services/scraper_backends/base.py @@ -91,7 +91,8 @@ class ScraperBackend(ABC): check_out: date, adults: int = 2, pages: int = 2, - dest_id: Optional[str] = None + dest_id: Optional[str] = None, + search_url: Optional[str] = None ) -> ScraperResult: """ Scrape booking.com location search results. @@ -105,6 +106,8 @@ class ScraperBackend(ABC): dest_id: Booking.com numeric destination id. Pins the search to one destination — free-text ss= resolves non-deterministically (Stow on the Wold intermittently matched St. Wolfgang, Austria) + search_url: A pasted Booking.com search URL whose ss/dest_id/ + dest_type pin the destination (wins over dest_id/location) Returns: ScraperResult with hotels and rates found diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index fc5d1cc..faeb28b 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -13,7 +13,24 @@ import re from datetime import date from decimal import Decimal, InvalidOperation from typing import List, Optional -from urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urlparse, parse_qs + + +def location_params_from_url(url: str) -> dict: + """Pull the stable destination params (ss, dest_id, dest_type) out of a + pasted Booking.com search URL, ignoring volatile session/click params and + the date/occupancy params we set ourselves per scrape.""" + if not url: + return {} + try: + q = parse_qs(urlparse(url).query) + except Exception: + return {} + out = {} + for key in ('ss', 'dest_id', 'dest_type'): + if q.get(key): + out[key] = q[key][0] + return out from playwright.async_api import async_playwright, Browser, BrowserContext, Page @@ -277,22 +294,30 @@ class PlaywrightLocalBackend(ScraperBackend): check_out: date, adults: int, offset: int = 0, - dest_id: Optional[str] = None + dest_id: Optional[str] = None, + location_params: Optional[dict] = None ) -> str: - """Build booking.com search URL with parameters.""" + """Build booking.com search URL. We always own the date/occupancy/paging + params; the destination comes from (in priority order) a pasted search + URL's params, an explicit dest_id, or the free-text location name.""" params = { - 'ss': location, 'checkin': check_in.isoformat(), 'checkout': check_out.isoformat(), 'group_adults': adults, 'no_rooms': 1, 'group_children': 0, } - if dest_id: - # Pin the destination — without this, free-text ss= intermittently - # resolves to the wrong place entirely. - params['dest_id'] = dest_id - params['dest_type'] = 'city' + if location_params: + # ss + dest_id + dest_type lifted from the user's pasted URL — + # the most reliable pin (it's exactly what their browser resolved). + params.update(location_params) + else: + params['ss'] = location + if dest_id: + # Pin the destination — without this, free-text ss= intermittently + # resolves to the wrong place entirely. + params['dest_id'] = dest_id + params['dest_type'] = 'city' if offset > 0: params['offset'] = offset @@ -509,7 +534,8 @@ class PlaywrightLocalBackend(ScraperBackend): check_out: date, adults: int = 2, pages: int = 2, - dest_id: Optional[str] = None + dest_id: Optional[str] = None, + search_url: Optional[str] = None ) -> ScraperResult: """ Scrape booking.com location search results, rotating the proxy IP if @@ -523,15 +549,19 @@ class PlaywrightLocalBackend(ScraperBackend): adults: Number of adults pages: Number of result pages to scrape dest_id: Booking.com numeric destination id (pins the search) + search_url: A pasted Booking.com search URL — its ss/dest_id/ + dest_type win over dest_id/location when present Returns: ScraperResult with hotels and rates found """ + location_params = location_params_from_url(search_url) if search_url else None max_attempts = 3 if self._proxy_enabled() else 1 result = None for attempt in range(max_attempts): - result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id) + result = await self._scrape_once( + location, check_in, check_out, adults, pages, dest_id, location_params) # Soft-block signals: an explicit challenge, or a "successful" load # that yielded zero hotels (page 1 never rendered results). @@ -556,7 +586,8 @@ class PlaywrightLocalBackend(ScraperBackend): check_out: date, adults: int, pages: int, - dest_id: Optional[str] = None + dest_id: Optional[str] = None, + location_params: Optional[dict] = None ) -> ScraperResult: """A single scrape attempt for one date on the current IP/session.""" all_hotels = [] @@ -583,7 +614,9 @@ class PlaywrightLocalBackend(ScraperBackend): page_loaded = True if page_num == 0: # Navigate to page 1 by URL - url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id) + url = self._build_search_url( + location, check_in, check_out, adults, + dest_id=dest_id, location_params=location_params) logger.info(f"Scraping page {page_num + 1}: {url}") try: await page.goto(url, wait_until='domcontentloaded', timeout=30000) @@ -598,7 +631,8 @@ class PlaywrightLocalBackend(ScraperBackend): # Fall back to offset URL if the control isn't found url = self._build_search_url( location, check_in, check_out, adults, - offset=page_num * 25, dest_id=dest_id + offset=page_num * 25, dest_id=dest_id, + location_params=location_params ) logger.info(f"Next-button not found, offset fallback: {url}") try: diff --git a/frontend/src/pages/MarketView.tsx b/frontend/src/pages/MarketView.tsx index 293edab..a5f039f 100644 --- a/frontend/src/pages/MarketView.tsx +++ b/frontend/src/pages/MarketView.tsx @@ -319,7 +319,8 @@ const StatusPanel: React.FC<{ status: ScraperStatus | undefined, isLoading: bool const SettingsTab: React.FC = () => { const queryClient = useQueryClient() const [locationName, setLocationName] = useState('') - const [pages, setPages] = useState(2) + const [searchUrl, setSearchUrl] = useState('') + const [pages, setPages] = useState(1) const [adults, setAdults] = useState(2) const [scrapeFrom, setScrapeFrom] = useState(() => fmtDate(new Date())) const [scrapeTo, setScrapeTo] = useState(() => { @@ -359,6 +360,7 @@ const SettingsTab: React.FC = () => { mutationFn: async () => { return (await api.post('/competitors/config/location', { location_name: locationName, + location_search_url: searchUrl || undefined, pages_to_scrape: pages, adults: adults, })).data @@ -366,6 +368,7 @@ const SettingsTab: React.FC = () => { onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['scraper-status'] }) setLocationName('') + setSearchUrl('') }, }) @@ -416,6 +419,23 @@ const SettingsTab: React.FC = () => { style={inputStyle} /> + +
+ + setSearchUrl(e.target.value)} + placeholder="https://www.booking.com/searchresults.html?ss=…&dest_id=…" + style={inputStyle} + /> +

+ Search your area on booking.com and paste the address-bar URL here. It pins the + exact destination (a plain name can resolve to the wrong place). Dates and paging + are set automatically — only the destination is read from the URL. +

+
+
{