From c49985bec7c817e9ba12caea99f9097bb038dd4a Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 10 Jul 2026 00:08:58 +0000 Subject: [PATCH] Add hotel discovery scrape, manual hotel add, and competitor-only filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Scraper: restrict hotel-page scraper to 'competitor' tier only (was own+competitor); own hotel rates come from Newbook API, not Booking.com - Backend: POST /competitors/discover — runs search-results scrape for one date to find market hotels regardless of current backend setting - Backend: POST /competitors/hotels — add hotel manually from Booking.com URL + name + tier; upserts on slug conflict - Frontend (Settings tab): Discover Market Hotels button added below manual date-range scrape - Frontend (Hotels tab): Add Hotel form at top — paste URL (auto-derives name from slug), choose tier, submit Co-Authored-By: Claude Sonnet 4.6 --- backend/api/competitors.py | 103 +++++++++++++++++++++ backend/services/booking_scraper.py | 51 +++++++++- frontend/src/pages/MarketView.tsx | 138 +++++++++++++++++++++++++++- 3 files changed, 287 insertions(+), 5 deletions(-) diff --git a/backend/api/competitors.py b/backend/api/competitors.py index c913661..d0a3ab2 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -452,10 +452,113 @@ async def reset_scraper( return result +@router.post("/discover") +async def trigger_discovery_scrape( + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user), +): + """ + Run a search-results scrape for one date to discover market hotels. + Always uses playwright_local (search-results) regardless of the configured + backend setting — intended for discovery, not rate tracking. + """ + if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])): + raise HTTPException(status_code=403, detail="manage_scraper capability required") + + location_result = await db.execute( + text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1") + ) + if not location_result.fetchone(): + raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.") + + from services.booking_scraper import get_lock_status + if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set(): + raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.") + + def _run(): + from services.booking_scraper import run_discovery_scrape + import asyncio + sync_db = SyncSessionLocal() + try: + asyncio.run(run_discovery_scrape(sync_db)) + finally: + sync_db.close() + _SCRAPE_PENDING.clear() + + _SCRAPE_PENDING.set() + background_tasks.add_task(_run) + return {"status": "started", "message": "Discovery scrape started. Check /status for progress."} + + # ============================================ # HOTELS MANAGEMENT # ============================================ + +class HotelManualCreate(BaseModel): + booking_com_url: str + name: str + tier: str = 'market' # 'own', 'competitor', 'market' + + +@router.post("/hotels", response_model=HotelResponse) +async def create_hotel_manually( + payload: HotelManualCreate, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user), +): + """Add a hotel manually by pasting its Booking.com URL.""" + if not (current_user.get('is_admin') or 'manage_hotels' in (current_user.get('caps') or [])): + raise HTTPException(status_code=403, detail="manage_hotels capability required") + + if payload.tier not in ('own', 'competitor', 'market'): + raise HTTPException(status_code=400, detail="tier must be own, competitor, or market") + + # Extract slug from URL as booking_com_id + import re + m = re.search(r'/hotel/\w+/([^.?#]+)', payload.booking_com_url) + slug = m.group(1) if m else None + if not slug: + raise HTTPException(status_code=400, detail="Could not parse hotel slug from URL. Expected a URL like booking.com/hotel/gb/hotel-name.en-gb.html") + + # Upsert — if slug already exists, update tier/url/name + result = await db.execute( + text(""" + INSERT INTO booking_com_hotels + (booking_com_id, name, booking_com_url, tier, is_active, first_seen_at, last_seen_at) + VALUES (:slug, :name, :url, :tier, TRUE, NOW(), NOW()) + ON CONFLICT (booking_com_id) DO UPDATE SET + name = EXCLUDED.name, + booking_com_url = EXCLUDED.booking_com_url, + tier = EXCLUDED.tier, + is_active = TRUE, + last_seen_at = NOW() + RETURNING id, booking_com_id, name, booking_com_url, star_rating, review_score, + review_count, tier, display_order, notes, first_seen_at, last_seen_at, + direct_hotel_id + """), + {'slug': slug, 'name': payload.name.strip(), 'url': payload.booking_com_url.strip(), 'tier': payload.tier} + ) + await db.commit() + row = result.fetchone() + return HotelResponse( + id=row.id, + booking_com_id=row.booking_com_id or '', + name=row.name, + booking_com_url=row.booking_com_url, + star_rating=float(row.star_rating) if row.star_rating else None, + review_score=float(row.review_score) if row.review_score else None, + review_count=row.review_count, + tier=row.tier, + display_order=row.display_order, + notes=row.notes, + first_seen_at=row.first_seen_at, + last_seen_at=row.last_seen_at, + direct_hotel_id=row.direct_hotel_id, + ) + + @router.get("/hotels", response_model=List[HotelResponse]) async def list_hotels( tier: Optional[str] = None, diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 5fdffce..b314056 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -295,16 +295,16 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60): def get_active_hotels(db: Session) -> List[Dict[str, Any]]: - """Return own + competitor hotels with a booking_com_url (for hotel-page scraping). - Market-tier hotels are excluded — they were auto-discovered from search results and - are not hotels we specifically want to track at rate-plan level.""" + """Return competitor hotels with a booking_com_url (for hotel-page scraping). + Own and market hotels are excluded — market were auto-discovered from search results; + own hotel rates come from the Newbook API, not Booking.com scraping.""" rows = db.execute( text(""" SELECT id, booking_com_id, name, booking_com_url FROM booking_com_hotels WHERE is_active = TRUE AND booking_com_url IS NOT NULL - AND tier IN ('own', 'competitor') + AND tier = 'competitor' ORDER BY display_order, id """) ).fetchall() @@ -884,6 +884,49 @@ async def _run_manual_scrape_locked( } +async def run_discovery_scrape(db: Session) -> Dict[str, Any]: + """ + Run a search-results scrape for a single date to discover market hotels. + Always uses playwright_local (search-results) regardless of the configured + backend — intended for hotel discovery, not rate tracking. + Returns new_hotels: count of hotels added to the DB this run. + """ + config = get_scrape_config(db) + if not config: + return {'success': False, 'error': 'No scrape location configured.'} + + hotels_before = db.execute( + text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE") + ).scalar() or 0 + + batch_id = create_scrape_batch(db, 'discovery') + check_in = date.today() + timedelta(days=14) + jobs: List[Tuple[date, Optional[int]]] = [(check_in, None)] + + try: + agg = await _scrape_dates_concurrent(jobs, config, concurrency=1, batch_id=batch_id) + update_scrape_batch( + db, batch_id, + status='completed' if agg['completed'] else 'failed', + hotels_found=agg['hotels'], + rates_scraped=agg['rates'], + ) + + hotels_after = db.execute( + text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE") + ).scalar() or 0 + + return { + 'success': agg['completed'] > 0, + 'hotels_found': agg['hotels'], + 'new_hotels': max(0, hotels_after - hotels_before), + } + except Exception as e: + logger.error(f"Discovery scrape error: {e}") + update_scrape_batch(db, batch_id, status='failed', error_message=str(e)) + return {'success': False, 'error': str(e)} + + # ============================================ # QUEUE MANAGEMENT # ============================================ diff --git a/frontend/src/pages/MarketView.tsx b/frontend/src/pages/MarketView.tsx index d743c0c..91a1e75 100644 --- a/frontend/src/pages/MarketView.tsx +++ b/frontend/src/pages/MarketView.tsx @@ -534,7 +534,7 @@ const SettingsTab: React.FC = () => {

Manual Scrape

- Trigger a one-off scrape for a date range. Runs in background. + Trigger a one-off competitor rate scrape for a date range. Runs in background.

@@ -579,6 +579,15 @@ const SettingsTab: React.FC = () => { {(scrapeMutation.error as any)?.response?.data?.detail || 'Failed to start scrape'}

)} + +
+ +

Discover Market Hotels

+

+ Runs the search-results scraper for one date to find hotels in your configured location. + Newly found hotels are added as Market tier so you can review and reclassify them. +

+
{/* Scrape History */} @@ -905,6 +914,131 @@ const ParityAlertsTab: React.FC = () => { ) } +const DiscoverButton: React.FC<{ locationConfigured: boolean }> = ({ locationConfigured }) => { + const qc = useQueryClient() + const m = useMutation({ + mutationFn: () => api.post('/competitors/discover').then(r => r.data), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ['scraper-status'] }) + qc.invalidateQueries({ queryKey: ['scrape-history'] }) + }, + }) + return ( +
+ + {!locationConfigured &&

Configure a location first

} + {m.isSuccess && ( +

+ Discovery started — new hotels will appear in the Hotels tab as Market tier. +

+ )} + {m.isError && ( +

+ {(m.error as any)?.response?.data?.detail || 'Discovery failed'} +

+ )} +
+ ) +} + + +const AddHotelForm: React.FC = () => { + const qc = useQueryClient() + const [url, setUrl] = useState('') + const [name, setName] = useState('') + const [tier, setTier] = useState('competitor') + + // Auto-derive a display name from the URL slug when the user pastes a URL + const handleUrlChange = (raw: string) => { + setUrl(raw) + const m = raw.match(/\/hotel\/\w+\/([^.?#]+)/) + if (m) { + const derived = m[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase()) + if (!name) setName(derived) + } + } + + const m = useMutation({ + mutationFn: () => api.post('/competitors/hotels', { booking_com_url: url.trim(), name: name.trim(), tier }).then(r => r.data), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ['competitor-hotels'] }) + setUrl('') + setName('') + setTier('competitor') + }, + }) + + return ( +
+
Add Hotel
+
+

+ Paste a Booking.com hotel page URL to add it directly without running a discovery scrape. +

+
+ + handleUrlChange(e.target.value)} + style={{ width: '100%' }} + /> +
+
+
+ + setName(e.target.value)} + style={{ width: '100%' }} + /> +
+
+ + +
+
+
+ + {m.isSuccess && ✓ Added} + {m.isError && ( + + {(m.error as any)?.response?.data?.detail || 'Failed to add hotel'} + + )} +
+
+
+ ) +} + + const HotelsTab: React.FC = () => { const queryClient = useQueryClient() const [tierFilter, setTierFilter] = useState('') @@ -1016,6 +1150,8 @@ const HotelsTab: React.FC = () => { return (
+ + {/* Filter */}
Filter: