Add hotel discovery scrape, manual hotel add, and competitor-only filtering
- Scraper: restrict hotel-page scraper to 'competitor' tier only (was own+competitor); own hotel rates come from Newbook API, not Booking.com - Backend: POST /competitors/discover — runs search-results scrape for one date to find market hotels regardless of current backend setting - Backend: POST /competitors/hotels — add hotel manually from Booking.com URL + name + tier; upserts on slug conflict - Frontend (Settings tab): Discover Market Hotels button added below manual date-range scrape - Frontend (Hotels tab): Add Hotel form at top — paste URL (auto-derives name from slug), choose tier, submit Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
30583c59a4
commit
c49985bec7
3 changed files with 287 additions and 5 deletions
|
|
@ -452,10 +452,113 @@ async def reset_scraper(
|
|||
return result
|
||||
|
||||
|
||||
@router.post("/discover")
|
||||
async def trigger_discovery_scrape(
|
||||
background_tasks: BackgroundTasks,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Run a search-results scrape for one date to discover market hotels.
|
||||
Always uses playwright_local (search-results) regardless of the configured
|
||||
backend setting — intended for discovery, not rate tracking.
|
||||
"""
|
||||
if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
|
||||
raise HTTPException(status_code=403, detail="manage_scraper capability required")
|
||||
|
||||
location_result = await db.execute(
|
||||
text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
|
||||
)
|
||||
if not location_result.fetchone():
|
||||
raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
|
||||
|
||||
from services.booking_scraper import get_lock_status
|
||||
if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set():
|
||||
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
|
||||
|
||||
def _run():
|
||||
from services.booking_scraper import run_discovery_scrape
|
||||
import asyncio
|
||||
sync_db = SyncSessionLocal()
|
||||
try:
|
||||
asyncio.run(run_discovery_scrape(sync_db))
|
||||
finally:
|
||||
sync_db.close()
|
||||
_SCRAPE_PENDING.clear()
|
||||
|
||||
_SCRAPE_PENDING.set()
|
||||
background_tasks.add_task(_run)
|
||||
return {"status": "started", "message": "Discovery scrape started. Check /status for progress."}
|
||||
|
||||
|
||||
# ============================================
|
||||
# HOTELS MANAGEMENT
|
||||
# ============================================
|
||||
|
||||
|
||||
class HotelManualCreate(BaseModel):
|
||||
booking_com_url: str
|
||||
name: str
|
||||
tier: str = 'market' # 'own', 'competitor', 'market'
|
||||
|
||||
|
||||
@router.post("/hotels", response_model=HotelResponse)
|
||||
async def create_hotel_manually(
|
||||
payload: HotelManualCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Add a hotel manually by pasting its Booking.com URL."""
|
||||
if not (current_user.get('is_admin') or 'manage_hotels' in (current_user.get('caps') or [])):
|
||||
raise HTTPException(status_code=403, detail="manage_hotels capability required")
|
||||
|
||||
if payload.tier not in ('own', 'competitor', 'market'):
|
||||
raise HTTPException(status_code=400, detail="tier must be own, competitor, or market")
|
||||
|
||||
# Extract slug from URL as booking_com_id
|
||||
import re
|
||||
m = re.search(r'/hotel/\w+/([^.?#]+)', payload.booking_com_url)
|
||||
slug = m.group(1) if m else None
|
||||
if not slug:
|
||||
raise HTTPException(status_code=400, detail="Could not parse hotel slug from URL. Expected a URL like booking.com/hotel/gb/hotel-name.en-gb.html")
|
||||
|
||||
# Upsert — if slug already exists, update tier/url/name
|
||||
result = await db.execute(
|
||||
text("""
|
||||
INSERT INTO booking_com_hotels
|
||||
(booking_com_id, name, booking_com_url, tier, is_active, first_seen_at, last_seen_at)
|
||||
VALUES (:slug, :name, :url, :tier, TRUE, NOW(), NOW())
|
||||
ON CONFLICT (booking_com_id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
booking_com_url = EXCLUDED.booking_com_url,
|
||||
tier = EXCLUDED.tier,
|
||||
is_active = TRUE,
|
||||
last_seen_at = NOW()
|
||||
RETURNING id, booking_com_id, name, booking_com_url, star_rating, review_score,
|
||||
review_count, tier, display_order, notes, first_seen_at, last_seen_at,
|
||||
direct_hotel_id
|
||||
"""),
|
||||
{'slug': slug, 'name': payload.name.strip(), 'url': payload.booking_com_url.strip(), 'tier': payload.tier}
|
||||
)
|
||||
await db.commit()
|
||||
row = result.fetchone()
|
||||
return HotelResponse(
|
||||
id=row.id,
|
||||
booking_com_id=row.booking_com_id or '',
|
||||
name=row.name,
|
||||
booking_com_url=row.booking_com_url,
|
||||
star_rating=float(row.star_rating) if row.star_rating else None,
|
||||
review_score=float(row.review_score) if row.review_score else None,
|
||||
review_count=row.review_count,
|
||||
tier=row.tier,
|
||||
display_order=row.display_order,
|
||||
notes=row.notes,
|
||||
first_seen_at=row.first_seen_at,
|
||||
last_seen_at=row.last_seen_at,
|
||||
direct_hotel_id=row.direct_hotel_id,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/hotels", response_model=List[HotelResponse])
|
||||
async def list_hotels(
|
||||
tier: Optional[str] = None,
|
||||
|
|
|
|||
|
|
@ -295,16 +295,16 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
|
|||
|
||||
|
||||
def get_active_hotels(db: Session) -> List[Dict[str, Any]]:
|
||||
"""Return own + competitor hotels with a booking_com_url (for hotel-page scraping).
|
||||
Market-tier hotels are excluded — they were auto-discovered from search results and
|
||||
are not hotels we specifically want to track at rate-plan level."""
|
||||
"""Return competitor hotels with a booking_com_url (for hotel-page scraping).
|
||||
Own and market hotels are excluded — market were auto-discovered from search results;
|
||||
own hotel rates come from the Newbook API, not Booking.com scraping."""
|
||||
rows = db.execute(
|
||||
text("""
|
||||
SELECT id, booking_com_id, name, booking_com_url
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE
|
||||
AND booking_com_url IS NOT NULL
|
||||
AND tier IN ('own', 'competitor')
|
||||
AND tier = 'competitor'
|
||||
ORDER BY display_order, id
|
||||
""")
|
||||
).fetchall()
|
||||
|
|
@ -884,6 +884,49 @@ async def _run_manual_scrape_locked(
|
|||
}
|
||||
|
||||
|
||||
async def run_discovery_scrape(db: Session) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a search-results scrape for a single date to discover market hotels.
|
||||
Always uses playwright_local (search-results) regardless of the configured
|
||||
backend — intended for hotel discovery, not rate tracking.
|
||||
Returns new_hotels: count of hotels added to the DB this run.
|
||||
"""
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
return {'success': False, 'error': 'No scrape location configured.'}
|
||||
|
||||
hotels_before = db.execute(
|
||||
text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE")
|
||||
).scalar() or 0
|
||||
|
||||
batch_id = create_scrape_batch(db, 'discovery')
|
||||
check_in = date.today() + timedelta(days=14)
|
||||
jobs: List[Tuple[date, Optional[int]]] = [(check_in, None)]
|
||||
|
||||
try:
|
||||
agg = await _scrape_dates_concurrent(jobs, config, concurrency=1, batch_id=batch_id)
|
||||
update_scrape_batch(
|
||||
db, batch_id,
|
||||
status='completed' if agg['completed'] else 'failed',
|
||||
hotels_found=agg['hotels'],
|
||||
rates_scraped=agg['rates'],
|
||||
)
|
||||
|
||||
hotels_after = db.execute(
|
||||
text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE")
|
||||
).scalar() or 0
|
||||
|
||||
return {
|
||||
'success': agg['completed'] > 0,
|
||||
'hotels_found': agg['hotels'],
|
||||
'new_hotels': max(0, hotels_after - hotels_before),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Discovery scrape error: {e}")
|
||||
update_scrape_batch(db, batch_id, status='failed', error_message=str(e))
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
# ============================================
|
||||
# QUEUE MANAGEMENT
|
||||
# ============================================
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue