hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Use pasted Booking.com search URL to pin scrape destination

Browse source 
The location_search_url column existed but was dead — the scraper always
rebuilt the URL from the free-text location name. Now the Location
Configuration form takes a "Booking.com search URL" field: paste the
address-bar URL from a real search and the scraper lifts ss/dest_id/
dest_type from it (the most reliable destination pin). Falls back to
dest_id, then plain name. Server also extracts dest_id from the URL for
the column and derives a display name from ss when none is typed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 22:00:28 +00:00
parent 9e5728efb1
commit 579a189cb8
5 changed files with 97 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

27
backend/api/competitors.py
View file

@ -27,10 +27,11 @@ class ScrapeRequest(BaseModel):
class LocationConfigRequest(BaseModel):
location_name: str
location_name: str = ''
pages_to_scrape: int = 2
adults: int = 2
dest_id: Optional[str] = None # Booking.com numeric destination id (pins the search)
location_search_url: Optional[str] = None # Pasted Booking.com search URL (pins destination)
class HotelTierUpdate(BaseModel):
@ -303,6 +304,19 @@ async def set_location_config(
current_user: dict = Depends(get_current_user)
):
"""Set the location to scrape for competitor rates."""
from services.scraper_backends.playwright_local import location_params_from_url
# If a full search URL was pasted, lift the destination params out of it —
# dest_id for the column (visibility/fallback), and ss for the display name
# when the user didn't type one.
search_url = (config.location_search_url or '').strip() or None
url_params = location_params_from_url(search_url) if search_url else {}
dest_id = config.dest_id or url_params.get('dest_id')
location_name = config.location_name.strip() or url_params.get('ss', '')
if not location_name:
raise HTTPException(status_code=400, detail="Provide a location name or a search URL.")
# Deactivate existing configs
await db.execute(
text("UPDATE booking_scrape_config SET is_active = FALSE")
@ -311,15 +325,16 @@ async def set_location_config(
# Insert new config
await db.execute(
text("""
INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, dest_id, is_active)
VALUES (:location, :pages, :adults, :dest_id, TRUE)
INSERT INTO booking_scrape_config
(location_name, pages_to_scrape, adults, dest_id, location_search_url, is_active)
VALUES (:location, :pages, :adults, :dest_id, :search_url, TRUE)
"""),
{'location': config.location_name, 'pages': config.pages_to_scrape,
'adults': config.adults, 'dest_id': config.dest_id}
{'location': location_name, 'pages': config.pages_to_scrape,
'adults': config.adults, 'dest_id': dest_id, 'search_url': search_url}
)
await db.commit()
return {"status": "success", "location": config.location_name}
return {"status": "success", "location": location_name}
@router.post("/config/enable")

1
backend/services/booking_scraper.py
View file

@ -333,6 +333,7 @@ async def scrape_date(
adults=config['adults'],
pages=config['pages_to_scrape'],
dest_id=config.get('dest_id'),
search_url=config.get('location_search_url'),
)
if result.blocked:

5
backend/services/scraper_backends/base.py
View file

@ -91,7 +91,8 @@ class ScraperBackend(ABC):
check_out: date,
adults: int = 2,
pages: int = 2,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
search_url: Optional[str] = None
) -> ScraperResult:
"""
Scrape booking.com location search results.
@ -105,6 +106,8 @@ class ScraperBackend(ABC):
dest_id: Booking.com numeric destination id. Pins the search to one
destination — free-text ss= resolves non-deterministically
(Stow on the Wold intermittently matched St. Wolfgang, Austria)
search_url: A pasted Booking.com search URL whose ss/dest_id/
dest_type pin the destination (wins over dest_id/location)
Returns:
ScraperResult with hotels and rates found

62
backend/services/scraper_backends/playwright_local.py
View file

@ -13,7 +13,24 @@ import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode, urlparse
from urllib.parse import urlencode, urlparse, parse_qs
def location_params_from_url(url: str) -> dict:
"""Pull the stable destination params (ss, dest_id, dest_type) out of a
pasted Booking.com search URL, ignoring volatile session/click params and
the date/occupancy params we set ourselves per scrape."""
if not url:
return {}
try:
q = parse_qs(urlparse(url).query)
except Exception:
return {}
out = {}
for key in ('ss', 'dest_id', 'dest_type'):
if q.get(key):
out[key] = q[key][0]
return out
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
@ -277,22 +294,30 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: date,
adults: int,
offset: int = 0,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
location_params: Optional[dict] = None
) -> str:
"""Build booking.com search URL with parameters."""
"""Build booking.com search URL. We always own the date/occupancy/paging
params; the destination comes from (in priority order) a pasted search
URL's params, an explicit dest_id, or the free-text location name."""
params = {
'ss': location,
'checkin': check_in.isoformat(),
'checkout': check_out.isoformat(),
'group_adults': adults,
'no_rooms': 1,
'group_children': 0,
}
if dest_id:
# Pin the destination — without this, free-text ss= intermittently
# resolves to the wrong place entirely.
params['dest_id'] = dest_id
params['dest_type'] = 'city'
if location_params:
# ss + dest_id + dest_type lifted from the user's pasted URL —
# the most reliable pin (it's exactly what their browser resolved).
params.update(location_params)
else:
params['ss'] = location
if dest_id:
# Pin the destination — without this, free-text ss= intermittently
# resolves to the wrong place entirely.
params['dest_id'] = dest_id
params['dest_type'] = 'city'
if offset > 0:
params['offset'] = offset
@ -509,7 +534,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: date,
adults: int = 2,
pages: int = 2,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
search_url: Optional[str] = None
) -> ScraperResult:
"""
Scrape booking.com location search results, rotating the proxy IP if
@ -523,15 +549,19 @@ class PlaywrightLocalBackend(ScraperBackend):
adults: Number of adults
pages: Number of result pages to scrape
dest_id: Booking.com numeric destination id (pins the search)
search_url: A pasted Booking.com search URL — its ss/dest_id/
dest_type win over dest_id/location when present
Returns:
ScraperResult with hotels and rates found
"""
location_params = location_params_from_url(search_url) if search_url else None
max_attempts = 3 if self._proxy_enabled() else 1
result = None
for attempt in range(max_attempts):
result = await self._scrape_once(location, check_in, check_out, adults, pages, dest_id)
result = await self._scrape_once(
location, check_in, check_out, adults, pages, dest_id, location_params)
# Soft-block signals: an explicit challenge, or a "successful" load
# that yielded zero hotels (page 1 never rendered results).
@ -556,7 +586,8 @@ class PlaywrightLocalBackend(ScraperBackend):
check_out: date,
adults: int,
pages: int,
dest_id: Optional[str] = None
dest_id: Optional[str] = None,
location_params: Optional[dict] = None
) -> ScraperResult:
"""A single scrape attempt for one date on the current IP/session."""
all_hotels = []
@ -583,7 +614,9 @@ class PlaywrightLocalBackend(ScraperBackend):
page_loaded = True
if page_num == 0:
# Navigate to page 1 by URL
url = self._build_search_url(location, check_in, check_out, adults, dest_id=dest_id)
url = self._build_search_url(
location, check_in, check_out, adults,
dest_id=dest_id, location_params=location_params)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
@ -598,7 +631,8 @@ class PlaywrightLocalBackend(ScraperBackend):
# Fall back to offset URL if the control isn't found
url = self._build_search_url(
location, check_in, check_out, adults,
offset=page_num * 25, dest_id=dest_id
offset=page_num * 25, dest_id=dest_id,
location_params=location_params
)
logger.info(f"Next-button not found, offset fallback: {url}")
try:

26
frontend/src/pages/MarketView.tsx
View file

@ -319,7 +319,8 @@ const StatusPanel: React.FC<{ status: ScraperStatus | undefined, isLoading: bool
const SettingsTab: React.FC = () => {
const queryClient = useQueryClient()
const [locationName, setLocationName] = useState('')
const [pages, setPages] = useState(2)
const [searchUrl, setSearchUrl] = useState('')
const [pages, setPages] = useState(1)
const [adults, setAdults] = useState(2)
const [scrapeFrom, setScrapeFrom] = useState(() => fmtDate(new Date()))
const [scrapeTo, setScrapeTo] = useState(() => {
@ -359,6 +360,7 @@ const SettingsTab: React.FC = () => {
mutationFn: async () => {
return (await api.post('/competitors/config/location', {
location_name: locationName,
location_search_url: searchUrl || undefined,
pages_to_scrape: pages,
adults: adults,
})).data
@ -366,6 +368,7 @@ const SettingsTab: React.FC = () => {
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
setLocationName('')
setSearchUrl('')
},
})
@ -416,6 +419,23 @@ const SettingsTab: React.FC = () => {
style={inputStyle}
/>
</div>
</div>
<div style={{ marginTop: '12px' }}>
<label style={inputLabelStyle}>Booking.com search URL (recommended)</label>
<input
type="text"
value={searchUrl}
onChange={e => setSearchUrl(e.target.value)}
placeholder="https://www.booking.com/searchresults.html?ss=…&dest_id=…"
style={inputStyle}
/>
<p style={{ fontSize: '12px', color: 'var(--text-mid)', margin: '6px 0 0' }}>
Search your area on booking.com and paste the address-bar URL here. It pins the
exact destination (a plain name can resolve to the wrong place). Dates and paging
are set automatically — only the destination is read from the URL.
</p>
</div>
<div style={{ ...styles.formRow, marginTop: '12px' }}>
<div style={styles.formGroupSmall}>
<label style={inputLabelStyle}>Pages</label>
<input
@ -441,10 +461,10 @@ const SettingsTab: React.FC = () => {
</div>
<button
onClick={() => setLocationMutation.mutate()}
disabled={!locationName || setLocationMutation.isPending}
disabled={(!locationName && !searchUrl) || setLocationMutation.isPending}
style={mergeStyles(
buttonStyle('primary'),
{ marginTop: '16px', opacity: !locationName ? 0.5 : 1 }
{ marginTop: '16px', opacity: (!locationName && !searchUrl) ? 0.5 : 1 }
)}
>
{setLocationMutation.isPending ? 'Saving...' : 'Set Location'}