Use pasted Booking.com search URL to pin scrape destination

The location_search_url column existed but was dead — the scraper always
rebuilt the URL from the free-text location name. Now the Location
Configuration form takes a "Booking.com search URL" field: paste the
address-bar URL from a real search and the scraper lifts ss/dest_id/
dest_type from it (the most reliable destination pin). Falls back to
dest_id, then plain name. Server also extracts dest_id from the URL for
the column and derives a display name from ss when none is typed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/api/competitors.py

@@ -27,10 +27,11 @@ class ScrapeRequest(BaseModel):
 
 
 class LocationConfigRequest(BaseModel):
-    location_name: str
+    location_name: str = ''
     pages_to_scrape: int = 2
     adults: int = 2
     dest_id: Optional[str] = None  # Booking.com numeric destination id (pins the search)
+    location_search_url: Optional[str] = None  # Pasted Booking.com search URL (pins destination)
 
 
 class HotelTierUpdate(BaseModel):
@@ -303,6 +304,19 @@ async def set_location_config(
     current_user: dict = Depends(get_current_user)
 ):
     """Set the location to scrape for competitor rates."""
+    from services.scraper_backends.playwright_local import location_params_from_url
+
+    # If a full search URL was pasted, lift the destination params out of it —
+    # dest_id for the column (visibility/fallback), and ss for the display name
+    # when the user didn't type one.
+    search_url = (config.location_search_url or '').strip() or None
+    url_params = location_params_from_url(search_url) if search_url else {}
+    dest_id = config.dest_id or url_params.get('dest_id')
+    location_name = config.location_name.strip() or url_params.get('ss', '')
+
+    if not location_name:
+        raise HTTPException(status_code=400, detail="Provide a location name or a search URL.")
+
     # Deactivate existing configs
     await db.execute(
         text("UPDATE booking_scrape_config SET is_active = FALSE")
@@ -311,15 +325,16 @@ async def set_location_config(
     # Insert new config
     await db.execute(
         text("""
-            INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, dest_id, is_active)
+            INSERT INTO booking_scrape_config
-            VALUES (:location, :pages, :adults, :dest_id, TRUE)
+                (location_name, pages_to_scrape, adults, dest_id, location_search_url, is_active)
+            VALUES (:location, :pages, :adults, :dest_id, :search_url, TRUE)
         """),
-        {'location': config.location_name, 'pages': config.pages_to_scrape,
+        {'location': location_name, 'pages': config.pages_to_scrape,
-         'adults': config.adults, 'dest_id': config.dest_id}
+         'adults': config.adults, 'dest_id': dest_id, 'search_url': search_url}
     )
     await db.commit()
 
-    return {"status": "success", "location": config.location_name}
+    return {"status": "success", "location": location_name}
 
 
 @router.post("/config/enable")

backend/services/booking_scraper.py

@@ -333,6 +333,7 @@ async def scrape_date(
         adults=config['adults'],
         pages=config['pages_to_scrape'],
         dest_id=config.get('dest_id'),
+        search_url=config.get('location_search_url'),
     )
 
     if result.blocked:

backend/services/scraper_backends/base.py

@@ -91,7 +91,8 @@ class ScraperBackend(ABC):
         check_out: date,
         adults: int = 2,
         pages: int = 2,
-        dest_id: Optional[str] = None
+        dest_id: Optional[str] = None,
+        search_url: Optional[str] = None
     ) -> ScraperResult:
         """
         Scrape booking.com location search results.
@@ -105,6 +106,8 @@ class ScraperBackend(ABC):
             dest_id: Booking.com numeric destination id. Pins the search to one
                 destination — free-text ss= resolves non-deterministically
                 (Stow on the Wold intermittently matched St. Wolfgang, Austria)
+            search_url: A pasted Booking.com search URL whose ss/dest_id/
+                dest_type pin the destination (wins over dest_id/location)
 
         Returns:
             ScraperResult with hotels and rates found

backend/services/scraper_backends/playwright_local.py

@@ -13,7 +13,24 @@ import re
 from datetime import date
 from decimal import Decimal, InvalidOperation
 from typing import List, Optional
-from urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode, urlparse, parse_qs
+
+
+def location_params_from_url(url: str) -> dict:
+    """Pull the stable destination params (ss, dest_id, dest_type) out of a
+    pasted Booking.com search URL, ignoring volatile session/click params and
+    the date/occupancy params we set ourselves per scrape."""
+    if not url:
+        return {}
+    try:
+        q = parse_qs(urlparse(url).query)
+    except Exception:
+        return {}
+    out = {}
+    for key in ('ss', 'dest_id', 'dest_type'):
+        if q.get(key):
+            out[key] = q[key][0]
+    return out
 
 from playwright.async_api import async_playwright, Browser, BrowserContext, Page
 
@@ -277,22 +294,30 @@ class PlaywrightLocalBackend(ScraperBackend):
         check_out: date,
         adults: int,
         offset: int = 0,
-        dest_id: Optional[str] = None
+        dest_id: Optional[str] = None,
+        location_params: Optional[dict] = None
     ) -> str:
-        """Build booking.com search URL with parameters."""
+        """Build booking.com search URL. We always own the date/occupancy/paging
+        params; the destination comes from (in priority order) a pasted search
+        URL's params, an explicit dest_id, or the free-text location name."""
         params = {
-            'ss': location,
             'checkin': check_in.isoformat(),
             'checkout': check_out.isoformat(),
             'group_adults': adults,
             'no_rooms': 1,
             'group_children': 0,
         }
-        if dest_id:
+        if location_params:
-            # Pin the destination — without this, free-text ss= intermittently
+            # ss + dest_id + dest_type lifted from the user's pasted URL —
-            # resolves to the wrong place entirely.
+            # the most reliable pin (it's exactly what their browser resolved).
-            params['dest_id'] = dest_id
+            params.update(location_params)
-            params['dest_type'] = 'city'
+        else:
+            params['ss'] = location
+            if dest_id:
+                # Pin the destination — without this, free-text ss= intermittently
+                # resolves to the wrong place entirely.
+                params['dest_id'] = dest_id
+                params['dest_type'] = 'city'
         if offset > 0:
             params['offset'] = offset
 
@@ -509,7 +534,8 @@ class PlaywrightLocalBackend(ScraperBackend):
         check_out: date,
         adults: int = 2,
         pages: int = 2,
-        dest_id: Optional[str] = None
+        dest_id: Optional[str] = None,
+        search_url: Optional[str] = None
     ) -> ScraperResult:
         """
         Scrape booking.com location search results, rotating the proxy IP if
@@ -523,15 +549,19 @@ class PlaywrightLocalBackend(ScraperBackend):
             adults: Number of adults
             pages: Number of result pages to scrape
             dest_id: Booking.com numeric destination id (pins the search)
+            search_url: A pasted Booking.com search URL — its ss/dest_id/
+                dest_type win over dest_id/location when present
 
         Returns:
             ScraperResult with hotels and rates found
         """
+        location_params = location_params_from_url(search_url) if search_url else None
         max_attempts = 3 if self._proxy_enabled() else 1
         result = None
 
         for attempt in range(max_attempts):
-            result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id)
+            result = await self._scrape_once(
+                location, check_in, check_out, adults, pages, dest_id, location_params)
 
             # Soft-block signals: an explicit challenge, or a "successful" load
             # that yielded zero hotels (page 1 never rendered results).
@@ -556,7 +586,8 @@ class PlaywrightLocalBackend(ScraperBackend):
         check_out: date,
         adults: int,
         pages: int,
-        dest_id: Optional[str] = None
+        dest_id: Optional[str] = None,
+        location_params: Optional[dict] = None
     ) -> ScraperResult:
         """A single scrape attempt for one date on the current IP/session."""
         all_hotels = []
@@ -583,7 +614,9 @@ class PlaywrightLocalBackend(ScraperBackend):
                 page_loaded = True
                 if page_num == 0:
                     # Navigate to page 1 by URL
-                    url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id)
+                    url = self._build_search_url(
+                        location, check_in, check_out, adults,
+                        dest_id=dest_id, location_params=location_params)
                     logger.info(f"Scraping page {page_num + 1}: {url}")
                     try:
                         await page.goto(url, wait_until='domcontentloaded', timeout=30000)
@@ -598,7 +631,8 @@ class PlaywrightLocalBackend(ScraperBackend):
                         # Fall back to offset URL if the control isn't found
                         url = self._build_search_url(
                             location, check_in, check_out, adults,
-                            offset=page_num * 25, dest_id=dest_id
+                            offset=page_num * 25, dest_id=dest_id,
+                            location_params=location_params
                         )
                         logger.info(f"Next-button not found, offset fallback: {url}")
                         try:

frontend/src/pages/MarketView.tsx

@@ -319,7 +319,8 @@ const StatusPanel: React.FC<{ status: ScraperStatus | undefined, isLoading: bool
 const SettingsTab: React.FC = () => {
   const queryClient = useQueryClient()
   const [locationName, setLocationName] = useState('')
-  const [pages, setPages] = useState(2)
+  const [searchUrl, setSearchUrl] = useState('')
+  const [pages, setPages] = useState(1)
   const [adults, setAdults] = useState(2)
   const [scrapeFrom, setScrapeFrom] = useState(() => fmtDate(new Date()))
   const [scrapeTo, setScrapeTo] = useState(() => {
@@ -359,6 +360,7 @@ const SettingsTab: React.FC = () => {
     mutationFn: async () => {
       return (await api.post('/competitors/config/location', {
         location_name: locationName,
+        location_search_url: searchUrl || undefined,
         pages_to_scrape: pages,
         adults: adults,
       })).data
@@ -366,6 +368,7 @@ const SettingsTab: React.FC = () => {
     onSuccess: () => {
       queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
       setLocationName('')
+      setSearchUrl('')
     },
   })
 
@@ -416,6 +419,23 @@ const SettingsTab: React.FC = () => {
               style={inputStyle}
             />
           </div>
+        </div>
+        <div style={{ marginTop: '12px' }}>
+          <label style={inputLabelStyle}>Booking.com search URL (recommended)</label>
+          <input
+            type="text"
+            value={searchUrl}
+            onChange={e => setSearchUrl(e.target.value)}
+            placeholder="https://www.booking.com/searchresults.html?ss=…&dest_id=…"
+            style={inputStyle}
+          />
+          <p style={{ fontSize: '12px', color: 'var(--text-mid)', margin: '6px 0 0' }}>
+            Search your area on booking.com and paste the address-bar URL here. It pins the
+            exact destination (a plain name can resolve to the wrong place). Dates and paging
+            are set automatically — only the destination is read from the URL.
+          </p>
+        </div>
+        <div style={{ ...styles.formRow, marginTop: '12px' }}>
           <div style={styles.formGroupSmall}>
             <label style={inputLabelStyle}>Pages</label>
             <input
@@ -441,10 +461,10 @@ const SettingsTab: React.FC = () => {
         </div>
         <button
           onClick={() => setLocationMutation.mutate()}
-          disabled={!locationName || setLocationMutation.isPending}
+          disabled={(!locationName && !searchUrl) || setLocationMutation.isPending}
           style={mergeStyles(
             buttonStyle('primary'),
-            { marginTop: '16px', opacity: !locationName ? 0.5 : 1 }
+            { marginTop: '16px', opacity: (!locationName && !searchUrl) ? 0.5 : 1 }
           )}
         >
           {setLocationMutation.isPending ? 'Saving...' : 'Set Location'}