Add hotel discovery scrape, manual hotel add, and competitor-only filtering
- Scraper: restrict hotel-page scraper to 'competitor' tier only (was own+competitor); own hotel rates come from Newbook API, not Booking.com - Backend: POST /competitors/discover — runs search-results scrape for one date to find market hotels regardless of current backend setting - Backend: POST /competitors/hotels — add hotel manually from Booking.com URL + name + tier; upserts on slug conflict - Frontend (Settings tab): Discover Market Hotels button added below manual date-range scrape - Frontend (Hotels tab): Add Hotel form at top — paste URL (auto-derives name from slug), choose tier, submit Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
30583c59a4
commit
c49985bec7
3 changed files with 287 additions and 5 deletions
|
|
@ -452,10 +452,113 @@ async def reset_scraper(
|
|||
return result
|
||||
|
||||
|
||||
@router.post("/discover")
|
||||
async def trigger_discovery_scrape(
|
||||
background_tasks: BackgroundTasks,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Run a search-results scrape for one date to discover market hotels.
|
||||
Always uses playwright_local (search-results) regardless of the configured
|
||||
backend setting — intended for discovery, not rate tracking.
|
||||
"""
|
||||
if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
|
||||
raise HTTPException(status_code=403, detail="manage_scraper capability required")
|
||||
|
||||
location_result = await db.execute(
|
||||
text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
|
||||
)
|
||||
if not location_result.fetchone():
|
||||
raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
|
||||
|
||||
from services.booking_scraper import get_lock_status
|
||||
if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set():
|
||||
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
|
||||
|
||||
def _run():
|
||||
from services.booking_scraper import run_discovery_scrape
|
||||
import asyncio
|
||||
sync_db = SyncSessionLocal()
|
||||
try:
|
||||
asyncio.run(run_discovery_scrape(sync_db))
|
||||
finally:
|
||||
sync_db.close()
|
||||
_SCRAPE_PENDING.clear()
|
||||
|
||||
_SCRAPE_PENDING.set()
|
||||
background_tasks.add_task(_run)
|
||||
return {"status": "started", "message": "Discovery scrape started. Check /status for progress."}
|
||||
|
||||
|
||||
# ============================================
|
||||
# HOTELS MANAGEMENT
|
||||
# ============================================
|
||||
|
||||
|
||||
class HotelManualCreate(BaseModel):
|
||||
booking_com_url: str
|
||||
name: str
|
||||
tier: str = 'market' # 'own', 'competitor', 'market'
|
||||
|
||||
|
||||
@router.post("/hotels", response_model=HotelResponse)
|
||||
async def create_hotel_manually(
|
||||
payload: HotelManualCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Add a hotel manually by pasting its Booking.com URL."""
|
||||
if not (current_user.get('is_admin') or 'manage_hotels' in (current_user.get('caps') or [])):
|
||||
raise HTTPException(status_code=403, detail="manage_hotels capability required")
|
||||
|
||||
if payload.tier not in ('own', 'competitor', 'market'):
|
||||
raise HTTPException(status_code=400, detail="tier must be own, competitor, or market")
|
||||
|
||||
# Extract slug from URL as booking_com_id
|
||||
import re
|
||||
m = re.search(r'/hotel/\w+/([^.?#]+)', payload.booking_com_url)
|
||||
slug = m.group(1) if m else None
|
||||
if not slug:
|
||||
raise HTTPException(status_code=400, detail="Could not parse hotel slug from URL. Expected a URL like booking.com/hotel/gb/hotel-name.en-gb.html")
|
||||
|
||||
# Upsert — if slug already exists, update tier/url/name
|
||||
result = await db.execute(
|
||||
text("""
|
||||
INSERT INTO booking_com_hotels
|
||||
(booking_com_id, name, booking_com_url, tier, is_active, first_seen_at, last_seen_at)
|
||||
VALUES (:slug, :name, :url, :tier, TRUE, NOW(), NOW())
|
||||
ON CONFLICT (booking_com_id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
booking_com_url = EXCLUDED.booking_com_url,
|
||||
tier = EXCLUDED.tier,
|
||||
is_active = TRUE,
|
||||
last_seen_at = NOW()
|
||||
RETURNING id, booking_com_id, name, booking_com_url, star_rating, review_score,
|
||||
review_count, tier, display_order, notes, first_seen_at, last_seen_at,
|
||||
direct_hotel_id
|
||||
"""),
|
||||
{'slug': slug, 'name': payload.name.strip(), 'url': payload.booking_com_url.strip(), 'tier': payload.tier}
|
||||
)
|
||||
await db.commit()
|
||||
row = result.fetchone()
|
||||
return HotelResponse(
|
||||
id=row.id,
|
||||
booking_com_id=row.booking_com_id or '',
|
||||
name=row.name,
|
||||
booking_com_url=row.booking_com_url,
|
||||
star_rating=float(row.star_rating) if row.star_rating else None,
|
||||
review_score=float(row.review_score) if row.review_score else None,
|
||||
review_count=row.review_count,
|
||||
tier=row.tier,
|
||||
display_order=row.display_order,
|
||||
notes=row.notes,
|
||||
first_seen_at=row.first_seen_at,
|
||||
last_seen_at=row.last_seen_at,
|
||||
direct_hotel_id=row.direct_hotel_id,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/hotels", response_model=List[HotelResponse])
|
||||
async def list_hotels(
|
||||
tier: Optional[str] = None,
|
||||
|
|
|
|||
|
|
@ -295,16 +295,16 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
|
|||
|
||||
|
||||
def get_active_hotels(db: Session) -> List[Dict[str, Any]]:
|
||||
"""Return own + competitor hotels with a booking_com_url (for hotel-page scraping).
|
||||
Market-tier hotels are excluded — they were auto-discovered from search results and
|
||||
are not hotels we specifically want to track at rate-plan level."""
|
||||
"""Return competitor hotels with a booking_com_url (for hotel-page scraping).
|
||||
Own and market hotels are excluded — market were auto-discovered from search results;
|
||||
own hotel rates come from the Newbook API, not Booking.com scraping."""
|
||||
rows = db.execute(
|
||||
text("""
|
||||
SELECT id, booking_com_id, name, booking_com_url
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE
|
||||
AND booking_com_url IS NOT NULL
|
||||
AND tier IN ('own', 'competitor')
|
||||
AND tier = 'competitor'
|
||||
ORDER BY display_order, id
|
||||
""")
|
||||
).fetchall()
|
||||
|
|
@ -884,6 +884,49 @@ async def _run_manual_scrape_locked(
|
|||
}
|
||||
|
||||
|
||||
async def run_discovery_scrape(db: Session) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a search-results scrape for a single date to discover market hotels.
|
||||
Always uses playwright_local (search-results) regardless of the configured
|
||||
backend — intended for hotel discovery, not rate tracking.
|
||||
Returns new_hotels: count of hotels added to the DB this run.
|
||||
"""
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
return {'success': False, 'error': 'No scrape location configured.'}
|
||||
|
||||
hotels_before = db.execute(
|
||||
text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE")
|
||||
).scalar() or 0
|
||||
|
||||
batch_id = create_scrape_batch(db, 'discovery')
|
||||
check_in = date.today() + timedelta(days=14)
|
||||
jobs: List[Tuple[date, Optional[int]]] = [(check_in, None)]
|
||||
|
||||
try:
|
||||
agg = await _scrape_dates_concurrent(jobs, config, concurrency=1, batch_id=batch_id)
|
||||
update_scrape_batch(
|
||||
db, batch_id,
|
||||
status='completed' if agg['completed'] else 'failed',
|
||||
hotels_found=agg['hotels'],
|
||||
rates_scraped=agg['rates'],
|
||||
)
|
||||
|
||||
hotels_after = db.execute(
|
||||
text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE")
|
||||
).scalar() or 0
|
||||
|
||||
return {
|
||||
'success': agg['completed'] > 0,
|
||||
'hotels_found': agg['hotels'],
|
||||
'new_hotels': max(0, hotels_after - hotels_before),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Discovery scrape error: {e}")
|
||||
update_scrape_batch(db, batch_id, status='failed', error_message=str(e))
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
# ============================================
|
||||
# QUEUE MANAGEMENT
|
||||
# ============================================
|
||||
|
|
|
|||
|
|
@ -534,7 +534,7 @@ const SettingsTab: React.FC = () => {
|
|||
<div style={styles.card}>
|
||||
<h3 style={styles.cardTitle}>Manual Scrape</h3>
|
||||
<p style={styles.cardDescription}>
|
||||
Trigger a one-off scrape for a date range. Runs in background.
|
||||
Trigger a one-off competitor rate scrape for a date range. Runs in background.
|
||||
</p>
|
||||
<div style={styles.formRow}>
|
||||
<div style={styles.formGroup}>
|
||||
|
|
@ -579,6 +579,15 @@ const SettingsTab: React.FC = () => {
|
|||
{(scrapeMutation.error as any)?.response?.data?.detail || 'Failed to start scrape'}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<hr style={{ margin: '20px 0', border: 'none', borderTop: '1px solid var(--border)' }} />
|
||||
|
||||
<h4 style={{ margin: '0 0 6px', fontSize: '14px', color: 'var(--text-dark)' }}>Discover Market Hotels</h4>
|
||||
<p style={styles.cardDescription}>
|
||||
Runs the search-results scraper for one date to find hotels in your configured location.
|
||||
Newly found hotels are added as <em>Market</em> tier so you can review and reclassify them.
|
||||
</p>
|
||||
<DiscoverButton locationConfigured={!!status?.location_configured} />
|
||||
</div>
|
||||
|
||||
{/* Scrape History */}
|
||||
|
|
@ -905,6 +914,131 @@ const ParityAlertsTab: React.FC = () => {
|
|||
)
|
||||
}
|
||||
|
||||
const DiscoverButton: React.FC<{ locationConfigured: boolean }> = ({ locationConfigured }) => {
|
||||
const qc = useQueryClient()
|
||||
const m = useMutation({
|
||||
mutationFn: () => api.post('/competitors/discover').then(r => r.data),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ['scraper-status'] })
|
||||
qc.invalidateQueries({ queryKey: ['scrape-history'] })
|
||||
},
|
||||
})
|
||||
return (
|
||||
<div>
|
||||
<button
|
||||
className="btn btn-outline"
|
||||
onClick={() => m.mutate()}
|
||||
disabled={m.isPending || !locationConfigured}
|
||||
style={{ opacity: locationConfigured ? 1 : 0.5 }}
|
||||
>
|
||||
{m.isPending ? 'Discovering…' : 'Discover Market Hotels'}
|
||||
</button>
|
||||
{!locationConfigured && <p style={styles.hintText}>Configure a location first</p>}
|
||||
{m.isSuccess && (
|
||||
<p style={{ color: 'var(--success)', fontSize: '13px', marginTop: '8px' }}>
|
||||
Discovery started — new hotels will appear in the Hotels tab as Market tier.
|
||||
</p>
|
||||
)}
|
||||
{m.isError && (
|
||||
<p style={{ color: 'var(--danger)', fontSize: '13px', marginTop: '8px' }}>
|
||||
{(m.error as any)?.response?.data?.detail || 'Discovery failed'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
const AddHotelForm: React.FC = () => {
|
||||
const qc = useQueryClient()
|
||||
const [url, setUrl] = useState('')
|
||||
const [name, setName] = useState('')
|
||||
const [tier, setTier] = useState('competitor')
|
||||
|
||||
// Auto-derive a display name from the URL slug when the user pastes a URL
|
||||
const handleUrlChange = (raw: string) => {
|
||||
setUrl(raw)
|
||||
const m = raw.match(/\/hotel\/\w+\/([^.?#]+)/)
|
||||
if (m) {
|
||||
const derived = m[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase())
|
||||
if (!name) setName(derived)
|
||||
}
|
||||
}
|
||||
|
||||
const m = useMutation({
|
||||
mutationFn: () => api.post('/competitors/hotels', { booking_com_url: url.trim(), name: name.trim(), tier }).then(r => r.data),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ['competitor-hotels'] })
|
||||
setUrl('')
|
||||
setName('')
|
||||
setTier('competitor')
|
||||
},
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="card" style={{ marginBottom: 16 }}>
|
||||
<div className="card-header">Add Hotel</div>
|
||||
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 12 }}>
|
||||
<p style={{ fontSize: 13, color: 'var(--text-mid)', margin: 0 }}>
|
||||
Paste a Booking.com hotel page URL to add it directly without running a discovery scrape.
|
||||
</p>
|
||||
<div>
|
||||
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
|
||||
Booking.com URL
|
||||
</label>
|
||||
<input
|
||||
type="url"
|
||||
placeholder="https://www.booking.com/hotel/gb/hotel-name.en-gb.html"
|
||||
value={url}
|
||||
onChange={e => handleUrlChange(e.target.value)}
|
||||
style={{ width: '100%' }}
|
||||
/>
|
||||
</div>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '1fr auto', gap: 12 }}>
|
||||
<div>
|
||||
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
|
||||
Display Name
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Hotel name"
|
||||
value={name}
|
||||
onChange={e => setName(e.target.value)}
|
||||
style={{ width: '100%' }}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
|
||||
Tier
|
||||
</label>
|
||||
<select value={tier} onChange={e => setTier(e.target.value)} style={{ width: 140 }}>
|
||||
<option value="competitor">Competitor</option>
|
||||
<option value="market">Market</option>
|
||||
<option value="own">Own Hotel</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
|
||||
<button
|
||||
className="btn btn-primary"
|
||||
onClick={() => m.mutate()}
|
||||
disabled={m.isPending || !url.trim() || !name.trim()}
|
||||
>
|
||||
{m.isPending ? 'Adding…' : 'Add Hotel'}
|
||||
</button>
|
||||
{m.isSuccess && <span style={{ fontSize: 12, color: 'var(--success)' }}>✓ Added</span>}
|
||||
{m.isError && (
|
||||
<span style={{ fontSize: 12, color: 'var(--danger)' }}>
|
||||
{(m.error as any)?.response?.data?.detail || 'Failed to add hotel'}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
const HotelsTab: React.FC = () => {
|
||||
const queryClient = useQueryClient()
|
||||
const [tierFilter, setTierFilter] = useState<string>('')
|
||||
|
|
@ -1016,6 +1150,8 @@ const HotelsTab: React.FC = () => {
|
|||
|
||||
return (
|
||||
<div>
|
||||
<AddHotelForm />
|
||||
|
||||
{/* Filter */}
|
||||
<div style={styles.filterRow}>
|
||||
<span style={styles.filterLabel}>Filter:</span>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue