hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add Rate Monitor app — Booking.com + direct booking engine competitor rates

Browse source 
Combines Booking.com Playwright scraper (from forecasting), direct booking
engine scraper (ported from laptop-archive/guestline-monitor), and Newbook
own-hotel rates into one focused tool. Four views: Bookability, Market View
(with price index badges + direct rate sub-rows), Direct Rates (per-competitor
room breakdown, min-stay flags, hotel config/discovery), Rate Analysis
(advance purchase curve, DOW chart, rate timeline, comparison table).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 12:06:30 +00:00
commit e05054172f
50 changed files with 11860 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
backend/api/__init__.py Normal file
View file

317
backend/api/analysis.py Normal file
View file

@ -0,0 +1,317 @@
"""
Rate Analysis API — advance purchase curves, DOW analysis, rate timelines, strategy summary
"""
from fastapi import APIRouter, Depends, Query
from sqlalchemy import text
from typing import Optional
from datetime import date, timedelta
from database import AsyncSessionLocal
from auth import get_current_user, require_cap
router = APIRouter()
async def get_db():
async with AsyncSessionLocal() as db:
yield db
# ─── Hotels available for analysis ───────────────────────────────────────────
@router.get("/hotels")
async def list_analysis_hotels(
tier: Optional[str] = Query(None, description="Filter by tier: own|competitor|market"),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
async with AsyncSessionLocal() as db:
where = "is_active = true"
params = {}
if tier:
where += " AND tier = :tier"
params["tier"] = tier
result = await db.execute(
text(f"""
SELECT h.id, h.name, h.tier, h.star_rating, h.review_score,
h.booking_com_url,
COUNT(DISTINCT r.rate_date) AS scraped_dates,
MAX(r.scraped_at) AS last_scraped
FROM booking_com_hotels h
LEFT JOIN booking_com_rates r ON r.hotel_id = h.id
WHERE {where}
GROUP BY h.id, h.name, h.tier, h.star_rating, h.review_score, h.booking_com_url
ORDER BY h.tier, h.display_order, h.name
"""),
params
)
rows = result.mappings().all()
return [dict(r) for r in rows]
# ─── Full analysis for one hotel ─────────────────────────────────────────────
@router.get("/hotel/{hotel_id}")
async def analyse_hotel(
hotel_id: int,
from_date: date = Query(default_factory=lambda: date.today()),
to_date: date = Query(default_factory=lambda: date.today() + timedelta(days=89)),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
async with AsyncSessionLocal() as db:
# Hotel info
hotel_row = await db.execute(
text("SELECT id, name, tier, star_rating, booking_com_url FROM booking_com_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = hotel_row.mappings().fetchone()
if not hotel:
from fastapi import HTTPException
raise HTTPException(status_code=404, detail="Hotel not found")
# Advance purchase curve
apc_result = await db.execute(
text("""
SELECT
CASE
WHEN (rate_date - scraped_at::date) <= 7 THEN '0-7d'
WHEN (rate_date - scraped_at::date) <= 29 THEN '8-29d'
WHEN (rate_date - scraped_at::date) <= 89 THEN '30-89d'
ELSE '90+d'
END AS lead_bucket,
ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate,
COUNT(*) AS sample_count
FROM booking_com_rates
WHERE hotel_id = :hid
AND rate_date BETWEEN :from_date AND :to_date
AND availability_status = 'available'
AND rate_gross IS NOT NULL
GROUP BY lead_bucket
ORDER BY
CASE lead_bucket
WHEN '0-7d' THEN 1
WHEN '8-29d' THEN 2
WHEN '30-89d' THEN 3
ELSE 4
END
"""),
{"hid": hotel_id, "from_date": from_date, "to_date": to_date}
)
advance_purchase_curve = [dict(r) for r in apc_result.mappings().all()]
# Day-of-week averages (latest scrape per date)
dow_result = await db.execute(
text("""
SELECT
EXTRACT(DOW FROM rate_date)::int AS dow,
TO_CHAR(rate_date, 'Dy') AS dow_label,
ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate,
COUNT(DISTINCT rate_date) AS date_count
FROM (
SELECT DISTINCT ON (rate_date)
rate_date, rate_gross
FROM booking_com_rates
WHERE hotel_id = :hid
AND rate_date BETWEEN :from_date AND :to_date
AND availability_status = 'available'
AND rate_gross IS NOT NULL
ORDER BY rate_date, scraped_at DESC
) latest
GROUP BY dow, dow_label
ORDER BY dow
"""),
{"hid": hotel_id, "from_date": from_date, "to_date": to_date}
)
dow_analysis = [dict(r) for r in dow_result.mappings().all()]
# Sold-out pattern by day-of-week
sold_out_result = await db.execute(
text("""
SELECT
EXTRACT(DOW FROM rate_date)::int AS dow,
TO_CHAR(rate_date, 'Dy') AS dow_label,
COUNT(DISTINCT rate_date) AS total_dates,
COUNT(DISTINCT rate_date) FILTER (
WHERE availability_status = 'sold_out'
) AS sold_out_dates
FROM (
SELECT DISTINCT ON (rate_date)
rate_date, availability_status
FROM booking_com_rates
WHERE hotel_id = :hid
AND rate_date BETWEEN :from_date AND :to_date
ORDER BY rate_date, scraped_at DESC
) latest
GROUP BY dow, dow_label
ORDER BY dow
"""),
{"hid": hotel_id, "from_date": from_date, "to_date": to_date}
)
sold_out_pattern = [dict(r) for r in sold_out_result.mappings().all()]
# Strategy summary
strategy = _compute_strategy(advance_purchase_curve, dow_analysis, sold_out_pattern)
return {
"hotel": dict(hotel),
"date_range": {"from": str(from_date), "to": str(to_date)},
"advance_purchase_curve": advance_purchase_curve,
"dow_analysis": dow_analysis,
"sold_out_pattern": sold_out_pattern,
"strategy_summary": strategy,
}
# ─── Rate timeline for a single date ─────────────────────────────────────────
@router.get("/hotel/{hotel_id}/timeline")
async def rate_timeline(
hotel_id: int,
rate_date: date = Query(...),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
async with AsyncSessionLocal() as db:
result = await db.execute(
text("""
SELECT
scraped_at,
rate_gross,
availability_status,
rooms_left,
room_type,
(rate_date - scraped_at::date) AS days_out
FROM booking_com_rates
WHERE hotel_id = :hid AND rate_date = :rd
ORDER BY scraped_at ASC
"""),
{"hid": hotel_id, "rd": rate_date}
)
rows = result.mappings().all()
return {
"hotel_id": hotel_id,
"rate_date": str(rate_date),
"timeline": [dict(r) for r in rows],
}
# ─── Own vs Competitor comparison ────────────────────────────────────────────
@router.get("/comparison")
async def rate_comparison(
competitor_ids: str = Query(..., description="Comma-separated hotel IDs to compare"),
from_date: date = Query(default_factory=lambda: date.today()),
to_date: date = Query(default_factory=lambda: date.today() + timedelta(days=29)),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
try:
comp_ids = [int(x.strip()) for x in competitor_ids.split(",") if x.strip()]
except ValueError:
from fastapi import HTTPException
raise HTTPException(status_code=400, detail="competitor_ids must be comma-separated integers")
async with AsyncSessionLocal() as db:
# Own hotel latest rates
own_result = await db.execute(
text("""
SELECT
ncr.rate_date,
ncr.gross_rate AS own_rate
FROM newbook_current_rates ncr
WHERE ncr.rate_date BETWEEN :from_date AND :to_date
ORDER BY ncr.rate_date
"""),
{"from_date": from_date, "to_date": to_date}
)
own_rates = {str(r.rate_date): float(r.own_rate) for r in own_result if r.own_rate}
# Competitor latest rates per date
comp_result = await db.execute(
text("""
SELECT
r.hotel_id,
h.name AS hotel_name,
r.rate_date,
r.rate_gross,
r.availability_status
FROM (
SELECT DISTINCT ON (hotel_id, rate_date)
hotel_id, rate_date, rate_gross, availability_status
FROM booking_com_rates
WHERE hotel_id = ANY(:comp_ids)
AND rate_date BETWEEN :from_date AND :to_date
ORDER BY hotel_id, rate_date, scraped_at DESC
) r
JOIN booking_com_hotels h ON h.id = r.hotel_id
ORDER BY r.rate_date, h.display_order
"""),
{"comp_ids": comp_ids, "from_date": from_date, "to_date": to_date}
)
comp_rows = comp_result.mappings().all()
# Build per-date rows
date_map: dict = {}
hotel_names: dict = {}
for row in comp_rows:
d = str(row["rate_date"])
if d not in date_map:
date_map[d] = {"date": d, "own_rate": own_rates.get(d)}
date_map[d][f"h{row['hotel_id']}"] = float(row["rate_gross"]) if row["rate_gross"] else None
date_map[d][f"h{row['hotel_id']}_status"] = row["availability_status"]
hotel_names[row["hotel_id"]] = row["hotel_name"]
return {
"hotel_names": hotel_names,
"rows": sorted(date_map.values(), key=lambda x: x["date"]),
}
# ─── Strategy computation helper ─────────────────────────────────────────────
def _compute_strategy(apc: list, dow: list, sold_out: list) -> dict:
# Advance discount — compare 0-7d vs 30-89d
rates_by_bucket = {r["lead_bucket"]: float(r["avg_rate"]) for r in apc if r.get("avg_rate")}
advance_discount_pct = None
if "0-7d" in rates_by_bucket and "30-89d" in rates_by_bucket:
close_in = rates_by_bucket["0-7d"]
far_out = rates_by_bucket["30-89d"]
if far_out > 0:
# Positive = closes in higher (scarcity premium); negative = discount for advance
advance_discount_pct = round((close_in - far_out) / far_out * 100, 1)
# Weekend premium — Fri(5)+Sat(6) vs Mon(1)–Thu(4)
rates_by_dow = {r["dow"]: float(r["avg_rate"]) for r in dow if r.get("avg_rate")}
weekend_premium_pct = None
weekend_rates = [rates_by_dow[d] for d in [5, 6] if d in rates_by_dow]
weekday_rates = [rates_by_dow[d] for d in [1, 2, 3, 4] if d in rates_by_dow]
if weekend_rates and weekday_rates:
avg_wk = sum(weekend_rates) / len(weekend_rates)
avg_wd = sum(weekday_rates) / len(weekday_rates)
if avg_wd > 0:
weekend_premium_pct = round((avg_wk - avg_wd) / avg_wd * 100, 1)
# Sold-out rate
total_dates = sum(r["total_dates"] for r in sold_out)
sold_out_dates = sum(r["sold_out_dates"] for r in sold_out)
sold_out_rate_pct = round(sold_out_dates / total_dates * 100, 1) if total_dates > 0 else None
# Strategy label
label = "Mixed / insufficient data"
if advance_discount_pct is not None:
if advance_discount_pct <= -5:
label = "Advance-booking discounter"
elif advance_discount_pct >= 8 and (sold_out_rate_pct or 0) >= 10:
label = "Yield manager (scarcity-driven)"
elif advance_discount_pct >= 3:
label = "Flat-rate / hold-firm strategy"
else:
label = "Stable pricing"
return {
"advance_discount_pct": advance_discount_pct,
"weekend_premium_pct": weekend_premium_pct,
"avg_sold_out_rate_pct": sold_out_rate_pct,
"strategy_label": label,
}

624
backend/api/bookability.py Normal file
View file

@ -0,0 +1,624 @@
"""
Bookability API endpoints
Rate availability matrix and competitor rate comparison
"""
from typing import Optional, List, Dict, Any
from datetime import date, datetime, timedelta
from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from pydantic import BaseModel
import logging
import json
from database import get_db, SyncSessionLocal
from auth import get_current_user
router = APIRouter()
logger = logging.getLogger(__name__)
# ============================================
# RESPONSE MODELS
# ============================================
class CategoryInfo(BaseModel):
category_id: str
category_name: str
room_count: int
class TariffInfo(BaseModel):
name: str
description: Optional[str] = None
rate: Optional[float] = None
average_nightly: Optional[float] = None
available: bool
message: str
sort_order: int = 999
min_stay: Optional[int] = None
available_for_min_stay: Optional[bool] = None # True if available when queried with min_stay nights
class OccupancyInfo(BaseModel):
occupied: int = 0
available: int = 0
maintenance: int = 0
class DateRateInfo(BaseModel):
rate_gross: Optional[float] = None
rate_net: Optional[float] = None
tariffs: List[TariffInfo]
tariff_count: int
occupancy: Optional[OccupancyInfo] = None
valid_from: Optional[str] = None
class RateMatrixResponse(BaseModel):
categories: List[CategoryInfo]
dates: List[str]
matrix: Dict[str, Dict[str, DateRateInfo]]
date_last_updated: Dict[str, Optional[str]] = {}
# ============================================
# RATE MATRIX ENDPOINT
# ============================================
@router.get("/rate-matrix", response_model=RateMatrixResponse)
async def get_rate_matrix(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
category_id: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate availability matrix for all tariffs across dates and categories.
Returns a matrix showing all available tariff options for each room category
and date combination, including availability status and rates.
Args:
from_date: Start date (YYYY-MM-DD), defaults to today
to_date: End date (YYYY-MM-DD), defaults to today + 30 days
category_id: Optional filter to specific category
Returns:
RateMatrixResponse with categories, dates, and the matrix data
"""
# Default date range
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Validate date range
if end < start:
raise HTTPException(status_code=400, detail="to_date must be after from_date")
if (end - start).days > 366:
raise HTTPException(status_code=400, detail="Date range cannot exceed 366 days")
# Fetch categories
cat_query = """
SELECT site_id, site_name, room_count
FROM newbook_room_categories
WHERE is_included = true
"""
params: Dict[str, Any] = {}
if category_id:
cat_query += " AND site_id = :category_id"
params["category_id"] = category_id
cat_query += " ORDER BY display_order, site_name"
cat_result = await db.execute(text(cat_query), params)
categories = [
CategoryInfo(
category_id=row.site_id,
category_name=row.site_name,
room_count=row.room_count or 0
)
for row in cat_result.fetchall()
]
if not categories:
return RateMatrixResponse(categories=[], dates=[], matrix={})
# Build date list
dates = []
current = start
while current <= end:
dates.append(current.isoformat())
current += timedelta(days=1)
# Fetch rates with tariffs_data (get latest version per category/date)
rates_query = """
SELECT DISTINCT ON (category_id, rate_date)
category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from
FROM newbook_current_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
"""
rates_params: Dict[str, Any] = {"from_date": start, "to_date": end}
if category_id:
rates_query += " AND category_id = :category_id"
rates_params["category_id"] = category_id
rates_query += " ORDER BY category_id, rate_date, valid_from DESC"
rates_result = await db.execute(text(rates_query), rates_params)
rates_rows = rates_result.fetchall()
# Fetch occupancy data from newbook_occupancy_report_data
occupancy_query = """
SELECT category_id, date, occupied, available, maintenance
FROM newbook_occupancy_report_data
WHERE date >= :from_date AND date <= :to_date
"""
occupancy_params: Dict[str, Any] = {"from_date": start, "to_date": end}
if category_id:
occupancy_query += " AND category_id = :category_id"
occupancy_params["category_id"] = category_id
occupancy_result = await db.execute(text(occupancy_query), occupancy_params)
occupancy_rows = occupancy_result.fetchall()
# Build occupancy lookup: category_id -> date -> OccupancyInfo
occupancy_map: Dict[str, Dict[str, OccupancyInfo]] = {}
for row in occupancy_rows:
cat_id = row.category_id
occ_date = row.date.isoformat()
if cat_id not in occupancy_map:
occupancy_map[cat_id] = {}
occupancy_map[cat_id][occ_date] = OccupancyInfo(
occupied=row.occupied or 0,
available=row.available or 0,
maintenance=row.maintenance or 0
)
# Build matrix
matrix: Dict[str, Dict[str, DateRateInfo]] = {}
# Initialize matrix with empty data for all categories and dates
for cat in categories:
matrix[cat.category_id] = {}
for date_str in dates:
# Get occupancy for this category/date if available
occ = occupancy_map.get(cat.category_id, {}).get(date_str)
matrix[cat.category_id][date_str] = DateRateInfo(
rate_gross=None,
rate_net=None,
tariffs=[],
tariff_count=0,
occupancy=occ
)
# Populate matrix with actual data
for row in rates_rows:
cat_id = row.category_id
rate_date = row.rate_date.isoformat()
if cat_id not in matrix or rate_date not in matrix[cat_id]:
continue
# Parse tariffs_data
tariffs_data = row.tariffs_data or {}
if isinstance(tariffs_data, str):
try:
tariffs_data = json.loads(tariffs_data)
except json.JSONDecodeError:
tariffs_data = {}
# Build tariff list
tariffs_list = []
raw_tariffs = tariffs_data.get('tariffs', [])
for idx, tariff in enumerate(raw_tariffs):
tariffs_list.append(TariffInfo(
name=tariff.get('name', 'Unknown'),
description=tariff.get('description'),
rate=tariff.get('rate'),
average_nightly=tariff.get('average_nightly'),
available=tariff.get('success', False),
message=tariff.get('message', ''),
sort_order=tariff.get('sort_order', idx),
min_stay=tariff.get('min_stay'),
available_for_min_stay=tariff.get('available_for_min_stay')
))
# Preserve existing occupancy data
existing_occ = matrix[cat_id][rate_date].occupancy
vf = row.valid_from.isoformat() if row.valid_from else None
matrix[cat_id][rate_date] = DateRateInfo(
rate_gross=float(row.rate_gross) if row.rate_gross else None,
rate_net=float(row.rate_net) if row.rate_net else None,
tariffs=tariffs_list,
tariff_count=tariffs_data.get('tariff_count', len(tariffs_list)),
occupancy=existing_occ,
valid_from=vf
)
# Per-date latest update time (max valid_from across all categories for each date)
date_last_updated: Dict[str, Optional[str]] = {}
for date_str in dates:
latest = None
for cat in categories:
vf = matrix.get(cat.category_id, {}).get(date_str, DateRateInfo(tariffs=[], tariff_count=0)).valid_from
if vf and (latest is None or vf > latest):
latest = vf
date_last_updated[date_str] = latest
return RateMatrixResponse(
categories=categories,
dates=dates,
matrix=matrix,
date_last_updated=date_last_updated
)
# ============================================
# RATE MATRIX SUMMARY (lightweight endpoint)
# ============================================
@router.get("/rate-matrix/summary")
async def get_rate_matrix_summary(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get a summary of rate availability issues.
Returns counts of unavailable tariffs by category and date for quick
identification of potential bookability problems.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Query rates with issues (get latest version per category/date)
result = await db.execute(
text("""
SELECT DISTINCT ON (category_id, rate_date)
category_id,
rate_date,
tariffs_data
FROM newbook_current_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
AND tariffs_data IS NOT NULL
ORDER BY category_id, rate_date, valid_from DESC
"""),
{"from_date": start, "to_date": end}
)
issues = []
for row in result.fetchall():
tariffs_data = row.tariffs_data or {}
if isinstance(tariffs_data, str):
try:
tariffs_data = json.loads(tariffs_data)
except json.JSONDecodeError:
continue
tariffs = tariffs_data.get('tariffs', [])
unavailable = [t for t in tariffs if not t.get('success', False)]
if unavailable:
issues.append({
"category_id": row.category_id,
"date": row.rate_date.isoformat(),
"unavailable_count": len(unavailable),
"unavailable_tariffs": [t.get('name') for t in unavailable],
"messages": [t.get('message') for t in unavailable if t.get('message')]
})
return {
"from_date": start.isoformat(),
"to_date": end.isoformat(),
"total_issues": len(issues),
"issues": issues
}
# ============================================
# RATE HISTORY ENDPOINT
# ============================================
@router.get("/rate-history/{category_id}/{rate_date}")
async def get_rate_history(
category_id: str,
rate_date: str,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate change history for a specific category and date.
Returns all rate snapshots showing how rates evolved over time.
Useful for understanding when rates changed and by how much.
"""
try:
target_date = date.fromisoformat(rate_date)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
result = await db.execute(
text("""
SELECT
id,
rate_gross,
rate_net,
tariffs_data,
valid_from,
last_verified_at
FROM newbook_current_rates
WHERE category_id = :category_id AND rate_date = :rate_date
ORDER BY valid_from DESC
"""),
{"category_id": category_id, "rate_date": target_date}
)
history = []
for row in result.fetchall():
tariffs_data = row.tariffs_data or {}
if isinstance(tariffs_data, str):
try:
tariffs_data = json.loads(tariffs_data)
except json.JSONDecodeError:
tariffs_data = {}
tariffs = tariffs_data.get('tariffs', [])
history.append({
"id": row.id,
"rate_gross": float(row.rate_gross) if row.rate_gross else None,
"rate_net": float(row.rate_net) if row.rate_net else None,
"valid_from": row.valid_from.isoformat() if row.valid_from else None,
"last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None,
"tariff_count": len(tariffs),
"tariffs_available": sum(1 for t in tariffs if t.get('success', False)),
"tariffs_unavailable": sum(1 for t in tariffs if not t.get('success', False)),
"tariffs": [
{
"name": t.get('name'),
"rate": t.get('rate'),
"available": t.get('success', False),
"message": t.get('message', ''),
"min_stay": t.get('min_stay')
}
for t in tariffs
]
})
return {
"category_id": category_id,
"rate_date": rate_date,
"version_count": len(history),
"history": history
}
# ============================================
# RATE CHANGES SUMMARY
# ============================================
@router.get("/rate-changes")
async def get_rate_changes(
days: int = 7,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get summary of rate changes in the last N days.
Shows which rates changed and when, useful for tracking pricing strategy changes.
"""
cutoff = datetime.now() - timedelta(days=days)
# Find dates with multiple versions (indicating changes)
result = await db.execute(
text("""
SELECT
category_id,
rate_date,
COUNT(*) as version_count,
MIN(valid_from) as first_version,
MAX(valid_from) as latest_version
FROM newbook_current_rates
WHERE valid_from >= :cutoff
GROUP BY category_id, rate_date
HAVING COUNT(*) > 1
ORDER BY MAX(valid_from) DESC
LIMIT 100
"""),
{"cutoff": cutoff}
)
changes = []
for row in result.fetchall():
changes.append({
"category_id": row.category_id,
"rate_date": row.rate_date.isoformat(),
"version_count": row.version_count,
"first_version": row.first_version.isoformat() if row.first_version else None,
"latest_version": row.latest_version.isoformat() if row.latest_version else None
})
return {
"days": days,
"total_changes": len(changes),
"changes": changes
}
# ============================================
# FETCH RATES TRIGGER (manual refresh)
# ============================================
@router.post("/refresh-rates")
async def refresh_rates(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
category_id: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Trigger a manual refresh of current rates from Newbook.
This runs the rate fetch job for the specified date range.
Note: This can be slow as it respects Newbook API rate limits.
"""
from jobs.fetch_current_rates import run_fetch_current_rates
# For now, just run the standard fetch
# TODO: Add support for custom date range and category filter
try:
await run_fetch_current_rates()
return {"status": "success", "message": "Rates refresh completed"}
except Exception as e:
logger.error(f"Rates refresh failed: {e}")
raise HTTPException(status_code=500, detail=f"Rates refresh failed: {str(e)}")
# ============================================
# SINGLE-DATE RATE REFRESH
# ============================================
def _refresh_date_sync(rate_date: date):
"""
Fetch rates for a single date from Newbook and save to DB.
Runs synchronously in a background task.
"""
import asyncio
from decimal import Decimal
from services.newbook_rates_client import NewbookRatesClient
from jobs.fetch_current_rates import save_rate_snapshot
db = SyncSessionLocal()
try:
# Get config
config_result = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')
""")
)
config = {row.config_key: row.config_value for row in config_result.fetchall()}
# Central Settings service first, app-local config fallback
from services.central_settings import get_newbook_credentials_sync
creds = get_newbook_credentials_sync()
if not creds:
if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
logger.error("Newbook credentials not configured for single-date refresh")
return
creds = {
'api_key': config['newbook_api_key'],
'username': config['newbook_username'],
'password': config['newbook_password'],
'region': config['newbook_region'],
}
vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20'))
# Get included categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = set(row.site_id for row in cat_result.fetchall())
client = NewbookRatesClient(
api_key=creds['api_key'],
username=creds['username'],
password=creds['password'],
region=creds['region'],
vat_rate=vat_rate
)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
async def _fetch():
async with client:
# Single-night query for this date (all categories)
category_rates = await client._fetch_all_categories_batch(
rate_date, guests_adults=2, guests_children=0
)
# Check for min_stay tariffs needing multi-night verification
dates_by_nights: Dict[int, list] = {}
for cat_id, rates in category_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
min_stay = tariff.get('min_stay')
if min_stay and min_stay > 1 and not tariff.get('success', False):
if min_stay not in dates_by_nights:
dates_by_nights[min_stay] = []
if rate_date not in dates_by_nights[min_stay]:
dates_by_nights[min_stay].append(rate_date)
# Run multi-night verification if needed
if dates_by_nights:
multi_results = await client.get_multi_night_availability(dates_by_nights)
for cat_id, rates in category_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
if rate_date in multi_results:
cat_avail = multi_results[rate_date].get(cat_id, {})
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
if tariff.get('min_stay') and tariff['min_stay'] > 1:
tariff['available_for_min_stay'] = cat_avail.get(tariff.get('name', ''), False)
# Save snapshots
inserted = 0
for cat_id, rates in category_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
result = save_rate_snapshot(db, cat_id, rate['date'], rate)
if result == 'inserted':
inserted += 1
return inserted
inserted = loop.run_until_complete(_fetch())
db.commit()
logger.info(f"Single-date refresh for {rate_date}: {inserted} new snapshots")
finally:
loop.close()
except Exception as e:
logger.error(f"Single-date refresh failed for {rate_date}: {e}", exc_info=True)
db.rollback()
finally:
db.close()
@router.post("/refresh-date/{rate_date}")
async def refresh_single_date(
rate_date: str,
current_user: dict = Depends(get_current_user)
):
"""
Trigger a refresh of rates for a single date from Newbook.
Runs synchronously so the client can invalidate its cache immediately on response.
"""
try:
target_date = date.fromisoformat(rate_date)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
import asyncio
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, _refresh_date_sync, target_date)
return {"status": "success", "date": rate_date, "message": f"Rates refreshed for {rate_date}"}

940
backend/api/competitors.py Normal file
View file

@ -0,0 +1,940 @@
"""
Competitor Rates API endpoints
Booking.com rate scraping, hotel management, and competitor comparison
"""
from typing import Optional, List, Dict, Any
from datetime import date, datetime, timedelta
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from pydantic import BaseModel
import logging
from database import get_db, SyncSessionLocal
from auth import get_current_user
router = APIRouter()
logger = logging.getLogger(__name__)
# ============================================
# REQUEST/RESPONSE MODELS
# ============================================
class ScrapeRequest(BaseModel):
from_date: str
to_date: Optional[str] = None
class LocationConfigRequest(BaseModel):
location_name: str
pages_to_scrape: int = 2
adults: int = 2
class HotelTierUpdate(BaseModel):
tier: str # 'own', 'competitor', 'market'
display_order: Optional[int] = None
class HotelResponse(BaseModel):
id: int
booking_com_id: str
name: str
booking_com_url: Optional[str]
star_rating: Optional[float]
review_score: Optional[float]
review_count: Optional[int]
tier: str
display_order: int
notes: Optional[str]
first_seen_at: Optional[datetime]
last_seen_at: Optional[datetime]
class RateResponse(BaseModel):
rate_date: str
hotel_id: int
hotel_name: str
tier: str
star_rating: Optional[float]
review_score: Optional[float]
availability_status: str
rate_gross: Optional[float]
room_type: Optional[str]
breakfast_included: Optional[bool]
free_cancellation: Optional[bool]
no_prepayment: Optional[bool]
rooms_left: Optional[int]
scraped_at: Optional[datetime]
class ScraperStatusResponse(BaseModel):
enabled: bool
paused: bool
pause_until: Optional[str]
backend: str
location_configured: bool
location_name: Optional[str]
last_scrape: Optional[dict]
# ============================================
# SCRAPER STATUS & CONFIGURATION
# ============================================
@router.get("/status", response_model=ScraperStatusResponse)
async def get_scraper_status(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get current scraper status and configuration."""
# Get config values
config_result = await db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN (
'booking_scraper_enabled',
'booking_scraper_paused',
'booking_scraper_pause_until',
'booking_scraper_backend'
)
""")
)
config = {row.config_key: row.config_value for row in config_result.fetchall()}
# Get location config
location_result = await db.execute(
text("SELECT location_name FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
)
location_row = location_result.fetchone()
# Get last scrape info
last_scrape_result = await db.execute(
text("""
SELECT batch_id, scrape_type, started_at, completed_at, status,
hotels_found, rates_scraped, error_message
FROM booking_scrape_log
ORDER BY started_at DESC
LIMIT 1
""")
)
last_scrape_row = last_scrape_result.fetchone()
last_scrape = None
if last_scrape_row:
last_scrape = {
'batch_id': str(last_scrape_row.batch_id),
'scrape_type': last_scrape_row.scrape_type,
'started_at': last_scrape_row.started_at.isoformat() if last_scrape_row.started_at else None,
'completed_at': last_scrape_row.completed_at.isoformat() if last_scrape_row.completed_at else None,
'status': last_scrape_row.status,
'hotels_found': last_scrape_row.hotels_found,
'rates_scraped': last_scrape_row.rates_scraped,
'error_message': last_scrape_row.error_message,
}
return ScraperStatusResponse(
enabled=config.get('booking_scraper_enabled', 'false') == 'true',
paused=config.get('booking_scraper_paused', 'false') == 'true',
pause_until=config.get('booking_scraper_pause_until'),
backend=config.get('booking_scraper_backend', 'playwright_local'),
location_configured=location_row is not None,
location_name=location_row.location_name if location_row else None,
last_scrape=last_scrape
)
@router.post("/config/location")
async def set_location_config(
config: LocationConfigRequest,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Set the location to scrape for competitor rates."""
# Deactivate existing configs
await db.execute(
text("UPDATE booking_scrape_config SET is_active = FALSE")
)
# Insert new config
await db.execute(
text("""
INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, is_active)
VALUES (:location, :pages, :adults, TRUE)
"""),
{'location': config.location_name, 'pages': config.pages_to_scrape, 'adults': config.adults}
)
await db.commit()
return {"status": "success", "location": config.location_name}
@router.post("/config/enable")
async def enable_scraper(
enabled: bool = True,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Enable or disable the booking.com scraper."""
await db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_enabled'"),
{'val': 'true' if enabled else 'false'}
)
await db.commit()
return {"status": "success", "enabled": enabled}
@router.post("/config/unpause")
async def unpause_scraper(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Manually unpause the scraper (clears blocking pause)."""
await db.execute(
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
)
await db.commit()
return {"status": "success", "message": "Scraper unpaused"}
# ============================================
# MANUAL SCRAPE TRIGGER
# ============================================
def run_scrape_sync(from_date: date, to_date: date):
"""Run scrape in sync context for background task."""
import asyncio
from services.booking_scraper import run_manual_scrape, cleanup_stale_batches
db = SyncSessionLocal()
try:
# Clean up any stale batches before starting
cleanup_stale_batches(db, max_age_minutes=60)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
result = loop.run_until_complete(run_manual_scrape(db, from_date, to_date))
logger.info(f"Background scrape completed: {result}")
finally:
loop.close()
except Exception as e:
logger.error(f"Background scrape failed: {e}", exc_info=True)
finally:
db.close()
@router.post("/scrape")
async def trigger_manual_scrape(
request: ScrapeRequest,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Trigger a manual scrape for the specified date range.
Runs in background - check /status for progress.
"""
try:
from_date = date.fromisoformat(request.from_date)
to_date = date.fromisoformat(request.to_date) if request.to_date else from_date
except ValueError as e:
raise HTTPException(status_code=400, detail=f"Invalid date format: {e}")
if to_date < from_date:
raise HTTPException(status_code=400, detail="to_date must be after from_date")
if (to_date - from_date).days > 30:
raise HTTPException(status_code=400, detail="Date range cannot exceed 30 days for manual scrape")
# Check if location is configured
location_result = await db.execute(
text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
)
if not location_result.fetchone():
raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
# Check if paused
paused_result = await db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
)
paused_row = paused_result.fetchone()
if paused_row and paused_row.config_value == 'true':
raise HTTPException(status_code=400, detail="Scraper is currently paused. Use /unpause first or wait for cooldown.")
# Start background task
background_tasks.add_task(run_scrape_sync, from_date, to_date)
return {
"status": "started",
"from_date": from_date.isoformat(),
"to_date": to_date.isoformat(),
"message": "Scrape started in background. Check /status for progress."
}
# ============================================
# HOTELS MANAGEMENT
# ============================================
@router.get("/hotels", response_model=List[HotelResponse])
async def list_hotels(
tier: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
List all discovered hotels.
Filter by tier: 'own', 'competitor', 'market', or None for all.
"""
query = """
SELECT id, booking_com_id, name, booking_com_url,
star_rating, review_score, review_count,
tier, display_order, notes, first_seen_at, last_seen_at
FROM booking_com_hotels
WHERE is_active = TRUE
"""
params = {}
if tier:
if tier not in ('own', 'competitor', 'market'):
raise HTTPException(status_code=400, detail="Invalid tier. Must be 'own', 'competitor', or 'market'")
query += " AND tier = :tier"
params['tier'] = tier
query += " ORDER BY display_order, name"
result = await db.execute(text(query), params)
return [
HotelResponse(
id=row.id,
booking_com_id=row.booking_com_id or '',
name=row.name,
booking_com_url=row.booking_com_url,
star_rating=float(row.star_rating) if row.star_rating else None,
review_score=float(row.review_score) if row.review_score else None,
review_count=row.review_count,
tier=row.tier,
display_order=row.display_order,
notes=row.notes,
first_seen_at=row.first_seen_at,
last_seen_at=row.last_seen_at
)
for row in result.fetchall()
]
@router.put("/hotels/{hotel_id}/tier")
async def update_hotel_tier(
hotel_id: int,
update: HotelTierUpdate,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Update a hotel's tier and display order.
Tiers:
- 'own': Your hotel (for parity checking)
- 'competitor': Main competitors (full tracking)
- 'market': Other hotels (context only)
"""
if update.tier not in ('own', 'competitor', 'market'):
raise HTTPException(status_code=400, detail="Invalid tier")
# If setting as 'own', clear any existing 'own' hotel
if update.tier == 'own':
await db.execute(
text("UPDATE booking_com_hotels SET tier = 'market' WHERE tier = 'own'")
)
# Update the hotel
set_clause = "tier = :tier"
params = {'hotel_id': hotel_id, 'tier': update.tier}
if update.display_order is not None:
set_clause += ", display_order = :order"
params['order'] = update.display_order
result = await db.execute(
text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id RETURNING id"),
params
)
if not result.fetchone():
raise HTTPException(status_code=404, detail="Hotel not found")
await db.commit()
# If this is now the own hotel, update system config
if update.tier == 'own':
await db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_own_hotel_id'"),
{'val': str(hotel_id)}
)
await db.commit()
return {"status": "success", "hotel_id": hotel_id, "tier": update.tier}
@router.put("/hotels/{hotel_id}/notes")
async def update_hotel_notes(
hotel_id: int,
notes: str,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Update notes for a hotel."""
result = await db.execute(
text("UPDATE booking_com_hotels SET notes = :notes WHERE id = :hotel_id RETURNING id"),
{'hotel_id': hotel_id, 'notes': notes}
)
if not result.fetchone():
raise HTTPException(status_code=404, detail="Hotel not found")
await db.commit()
return {"status": "success"}
# ============================================
# COMPETITOR RATES MATRIX
# ============================================
@router.get("/matrix")
async def get_competitor_matrix(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
include_market: bool = False,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate comparison matrix for competitors.
Returns rates for own hotel and competitors, organized by date.
Set include_market=true to also include market tier hotels.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
if end < start:
raise HTTPException(status_code=400, detail="to_date must be after from_date")
if (end - start).days > 90:
raise HTTPException(status_code=400, detail="Date range cannot exceed 90 days")
tier_filter = "h.tier IN ('own', 'competitor')"
if include_market:
tier_filter = "h.tier IN ('own', 'competitor', 'market')"
# Get hotels
hotels_result = await db.execute(
text(f"""
SELECT id, name, tier, display_order, star_rating, review_score, booking_com_url
FROM booking_com_hotels
WHERE is_active = TRUE AND {tier_filter.replace('h.', '')}
ORDER BY display_order, name
""")
)
hotels = [dict(row._mapping) for row in hotels_result.fetchall()]
# Get latest rates using the view
rates_result = await db.execute(
text(f"""
SELECT DISTINCT ON (r.hotel_id, r.rate_date)
r.hotel_id,
r.rate_date,
r.availability_status,
r.rate_gross,
r.room_type,
r.breakfast_included,
r.free_cancellation,
r.no_prepayment,
r.rooms_left,
r.scraped_at
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE {tier_filter}
AND h.is_active = TRUE
AND r.rate_date >= :from_date AND r.rate_date <= :to_date
ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC
"""),
{'from_date': start, 'to_date': end}
)
# Build matrix: hotel_id -> date -> rate data
rates_by_hotel: Dict[int, Dict[str, dict]] = {}
for row in rates_result.fetchall():
hotel_id = row.hotel_id
rate_date = row.rate_date.isoformat()
if hotel_id not in rates_by_hotel:
rates_by_hotel[hotel_id] = {}
rates_by_hotel[hotel_id][rate_date] = {
'availability_status': row.availability_status,
'rate_gross': float(row.rate_gross) if row.rate_gross else None,
'room_type': row.room_type,
'breakfast_included': row.breakfast_included,
'free_cancellation': row.free_cancellation,
'no_prepayment': row.no_prepayment,
'rooms_left': row.rooms_left,
'scraped_at': row.scraped_at.isoformat() if row.scraped_at else None,
}
# Build date list
dates = []
current = start
while current <= end:
dates.append(current.isoformat())
current += timedelta(days=1)
return {
'from_date': start.isoformat(),
'to_date': end.isoformat(),
'dates': dates,
'hotels': hotels,
'rates': rates_by_hotel
}
# ============================================
# RATE PARITY (OWN HOTEL VS NEWBOOK)
# ============================================
@router.get("/parity")
async def get_rate_parity(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate parity comparison between booking.com and Newbook rates.
Compares scraped booking.com rates for own hotel against Newbook current rates.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Get own hotel's booking.com rates
booking_rates_result = await db.execute(
text("""
SELECT DISTINCT ON (r.rate_date)
r.rate_date,
r.rate_gross as booking_rate,
r.availability_status,
r.room_type as booking_room_type,
r.scraped_at
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE h.tier = 'own'
AND r.rate_date >= :from_date AND r.rate_date <= :to_date
ORDER BY r.rate_date, r.scraped_at DESC
"""),
{'from_date': start, 'to_date': end}
)
booking_rates = {row.rate_date: dict(row._mapping) for row in booking_rates_result.fetchall()}
# Get Newbook rates (best rate per date across categories)
newbook_rates_result = await db.execute(
text("""
SELECT DISTINCT ON (rate_date)
rate_date,
rate_gross as newbook_rate,
category_id
FROM newbook_current_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
ORDER BY rate_date, valid_from DESC
"""),
{'from_date': start, 'to_date': end}
)
newbook_rates = {row.rate_date: dict(row._mapping) for row in newbook_rates_result.fetchall()}
# Compare rates
parity_issues = []
all_dates = set(booking_rates.keys()) | set(newbook_rates.keys())
for rate_date in sorted(all_dates):
booking = booking_rates.get(rate_date)
newbook = newbook_rates.get(rate_date)
if not booking or not newbook:
continue
booking_rate = booking.get('booking_rate')
newbook_rate = newbook.get('newbook_rate')
if not booking_rate or not newbook_rate:
continue
diff_pct = ((float(booking_rate) - float(newbook_rate)) / float(newbook_rate)) * 100
if abs(diff_pct) > 1: # More than 1% difference
parity_issues.append({
'rate_date': rate_date.isoformat(),
'booking_rate': float(booking_rate),
'newbook_rate': float(newbook_rate),
'difference_pct': round(diff_pct, 2),
'alert_type': 'higher' if diff_pct > 0 else 'lower',
'booking_room_type': booking.get('booking_room_type'),
'availability_status': booking.get('availability_status'),
})
return {
'from_date': start.isoformat(),
'to_date': end.isoformat(),
'issues_count': len(parity_issues),
'issues': parity_issues
}
# ============================================
# PARITY ALERTS
# ============================================
@router.get("/parity/alerts")
async def get_parity_alerts(
status: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get rate parity alerts."""
query = """
SELECT id, rate_date, room_category, newbook_rate, booking_com_rate,
difference_pct, alert_type, alert_status, created_at,
acknowledged_at, acknowledged_by, notes
FROM rate_parity_alerts
"""
params = {}
if status:
query += " WHERE alert_status = :status"
params['status'] = status
query += " ORDER BY rate_date DESC, created_at DESC LIMIT 100"
result = await db.execute(text(query), params)
return [
{
'id': row.id,
'rate_date': row.rate_date.isoformat(),
'room_category': row.room_category,
'newbook_rate': float(row.newbook_rate) if row.newbook_rate else None,
'booking_com_rate': float(row.booking_com_rate) if row.booking_com_rate else None,
'difference_pct': float(row.difference_pct) if row.difference_pct else None,
'alert_type': row.alert_type,
'alert_status': row.alert_status,
'created_at': row.created_at.isoformat() if row.created_at else None,
'acknowledged_at': row.acknowledged_at.isoformat() if row.acknowledged_at else None,
'acknowledged_by': row.acknowledged_by,
'notes': row.notes,
}
for row in result.fetchall()
]
@router.put("/parity/alerts/{alert_id}/acknowledge")
async def acknowledge_parity_alert(
alert_id: int,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Acknowledge a parity alert."""
result = await db.execute(
text("""
UPDATE rate_parity_alerts
SET alert_status = 'acknowledged',
acknowledged_at = NOW(),
acknowledged_by = :username
WHERE id = :alert_id
RETURNING id
"""),
{'alert_id': alert_id, 'username': current_user.get('username', 'unknown')}
)
if not result.fetchone():
raise HTTPException(status_code=404, detail="Alert not found")
await db.commit()
return {"status": "success"}
# ============================================
# QUEUE STATUS
# ============================================
@router.get("/queue-status")
async def get_queue_status(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get current scrape queue status."""
result = await db.execute(
text("""
SELECT
status,
COUNT(*) as count,
MIN(rate_date) as earliest_date,
MAX(rate_date) as latest_date
FROM booking_scrape_queue
GROUP BY status
""")
)
status_counts = {row.status: {
'count': row.count,
'earliest': row.earliest_date.isoformat() if row.earliest_date else None,
'latest': row.latest_date.isoformat() if row.latest_date else None,
} for row in result.fetchall()}
# Get retry items (failed but under max_attempts)
retry_result = await db.execute(
text("""
SELECT COUNT(*) as count
FROM booking_scrape_queue
WHERE status = 'pending' AND attempts > 0
""")
)
retry_count = retry_result.fetchone().count
return {
'statuses': status_counts,
'retries_pending': retry_count,
'total_pending': status_counts.get('pending', {}).get('count', 0),
'total_completed': status_counts.get('completed', {}).get('count', 0),
'total_failed': status_counts.get('failed', {}).get('count', 0),
}
# ============================================
# SCHEDULE INFO
# ============================================
@router.get("/schedule-info")
async def get_schedule_info(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get information about the scraping schedule."""
# Get configured time
time_result = await db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_daily_time'")
)
time_row = time_result.fetchone()
daily_time = time_row.config_value if time_row and time_row.config_value else '05:30'
# Calculate what today's schedule would look like
from jobs.scrape_booking_rates import get_high_priority_dates, get_medium_priority_dates, get_low_priority_dates
high = get_high_priority_dates()
medium = get_medium_priority_dates()
low = get_low_priority_dates()
today = date.today()
weekday_name = today.strftime('%A')
return {
'daily_time': daily_time,
'today': today.isoformat(),
'weekday': weekday_name,
'tiers': {
'high': {
'description': 'Next 30 days (scraped first)',
'dates_today': len(high),
'range': f'{high[0].isoformat()} to {high[-1].isoformat()}' if high else None,
},
'medium': {
'description': 'Days 31-180 (scraped after high priority)',
'dates_today': len(medium),
'range': f'{medium[0].isoformat()} to {medium[-1].isoformat()}' if medium else None,
},
'low': {
'description': 'Days 181-365 (scraped last, or until rate limit)',
'dates_today': len(low),
'range': f'{low[0].isoformat()} to {low[-1].isoformat()}' if low else None,
},
},
'total_dates_today': len(set(high + medium + low)),
}
# ============================================
# SCRAPE COVERAGE (365-day view)
# ============================================
@router.get("/scrape-coverage")
async def get_scrape_coverage(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get 365-day scrape coverage showing last scraped time
and next expected scrape for every date.
"""
today = date.today()
end = today + timedelta(days=365)
# Get latest scraped_at per date (across all hotels)
result = await db.execute(
text("""
SELECT rate_date, MAX(scraped_at) as last_scraped
FROM booking_com_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
GROUP BY rate_date
"""),
{'from_date': today, 'to_date': end}
)
scraped_map = {row.rate_date: row.last_scraped for row in result.fetchall()}
# Compute tier and next scrape for each date
from jobs.scrape_booking_rates import compute_next_scrape_for_date
coverage = []
for offset in range(366):
d = today + timedelta(days=offset)
tier, next_scrape = compute_next_scrape_for_date(d)
last_scraped = scraped_map.get(d)
coverage.append({
'date': d.isoformat(),
'tier': tier,
'last_scraped': last_scraped.isoformat() if last_scraped else None,
'next_expected': next_scrape.isoformat() if next_scrape else None,
})
return {
'today': today.isoformat(),
'coverage': coverage,
}
# ============================================
# BOOKING.COM AVAILABILITY CHECK (for Bookability page)
# ============================================
@router.get("/booking-availability")
async def get_booking_availability(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Check own hotel's availability on booking.com.
Returns a simple summary: for each date in the range, whether the own hotel
appears available on booking.com based on the latest scrape data.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Get own hotel's latest scraped availability
result = await db.execute(
text("""
SELECT DISTINCT ON (r.rate_date)
r.rate_date,
r.availability_status,
r.rate_gross,
r.scraped_at
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE h.tier = 'own'
AND r.rate_date >= :from_date AND r.rate_date <= :to_date
ORDER BY r.rate_date, r.scraped_at DESC
"""),
{'from_date': start, 'to_date': end}
)
rows = result.fetchall()
if not rows:
return {
'has_own_hotel': False,
'dates_checked': 0,
'dates_available': 0,
'dates_sold_out': 0,
'dates_no_data': 0,
'latest_scrape': None,
'dates': {},
}
dates_map = {}
dates_available = 0
dates_sold_out = 0
dates_no_data = 0
latest_scrape = None
for row in rows:
status = row.availability_status
dates_map[row.rate_date.isoformat()] = {
'status': status,
'rate': float(row.rate_gross) if row.rate_gross else None,
}
if status == 'available':
dates_available += 1
elif status == 'sold_out':
dates_sold_out += 1
else:
dates_no_data += 1
if row.scraped_at and (not latest_scrape or row.scraped_at > latest_scrape):
latest_scrape = row.scraped_at
return {
'has_own_hotel': True,
'dates_checked': len(rows),
'dates_available': dates_available,
'dates_sold_out': dates_sold_out,
'dates_no_data': dates_no_data,
'latest_scrape': latest_scrape.isoformat() if latest_scrape else None,
'dates': dates_map,
}
# ============================================
# SCRAPE HISTORY
# ============================================
@router.get("/scrape-history")
async def get_scrape_history(
limit: int = 20,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get recent scrape batch history."""
result = await db.execute(
text("""
SELECT batch_id, scrape_type, started_at, completed_at, status,
dates_queued, dates_completed, dates_failed,
hotels_found, rates_scraped, error_message,
blocked_at, resume_after
FROM booking_scrape_log
ORDER BY started_at DESC
LIMIT :limit
"""),
{'limit': limit}
)
return [
{
'batch_id': str(row.batch_id),
'scrape_type': row.scrape_type,
'started_at': row.started_at.isoformat() if row.started_at else None,
'completed_at': row.completed_at.isoformat() if row.completed_at else None,
'status': row.status,
'dates_queued': row.dates_queued,
'dates_completed': row.dates_completed,
'dates_failed': row.dates_failed,
'hotels_found': row.hotels_found,
'rates_scraped': row.rates_scraped,
'error_message': row.error_message,
'blocked_at': row.blocked_at.isoformat() if row.blocked_at else None,
'resume_after': row.resume_after.isoformat() if row.resume_after else None,
}
for row in result.fetchall()
]

337
backend/api/direct.py Normal file
View file

@ -0,0 +1,337 @@
"""
Direct booking engine API — hotel management, discovery, scrape control, rate data.
"""
import asyncio
import json
import logging
from datetime import date, timedelta
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from pydantic import BaseModel
from sqlalchemy import text
from database import AsyncSessionLocal, SyncSessionLocal
from auth import get_current_user, require_cap
from services.direct_profiles import PROFILES, detect_profile, get_profile
from services.direct_scraper import run_discovery, get_discovery_status, run_scrape
router = APIRouter()
log = logging.getLogger(__name__)
# ─── Pydantic models ─────────────────────────────────────────────────────────
class HotelCreate(BaseModel):
name: str
profile_name: str
params: dict
class HotelUpdate(BaseModel):
name: Optional[str] = None
room_labels: Optional[dict] = None
rate_labels: Optional[dict] = None
room_order: Optional[list] = None
benchmark_room: Optional[str] = None
benchmark_rate: Optional[str] = None
tier_base_room: Optional[str] = None
tier_offsets: Optional[dict] = None
scrape_enabled: Optional[bool] = None
params: Optional[dict] = None
class DetectRequest(BaseModel):
url: str
# ─── Profiles ────────────────────────────────────────────────────────────────
@router.get("/profiles")
async def list_profiles(user=Depends(get_current_user)):
require_cap(user, "view_direct_rates")
return [
{
"name": name,
"label": cls.label,
"required_params": cls.required_params,
}
for name, cls in PROFILES.items()
]
@router.post("/profiles/detect")
async def detect_engine(req: DetectRequest, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
result = detect_profile(req.url)
if not result:
raise HTTPException(status_code=422, detail="Could not detect booking engine from URL")
return result
# ─── Hotel CRUD ───────────────────────────────────────────────────────────────
@router.get("/hotels")
async def list_hotels(user=Depends(get_current_user)):
require_cap(user, "view_direct_rates")
async with AsyncSessionLocal() as db:
result = await db.execute(
text("""SELECT h.id, h.name, h.profile_name, h.params,
h.room_labels, h.rate_labels, h.room_order,
h.benchmark_room, h.benchmark_rate, h.tier_base_room, h.tier_offsets,
h.scrape_enabled, h.last_scraped_at,
COUNT(DISTINCT r.stay_date) AS scraped_dates,
MAX(r.scraped_at) AS last_rate_at
FROM direct_competitor_hotels h
LEFT JOIN direct_rates r ON r.hotel_id = h.id
GROUP BY h.id
ORDER BY h.id""")
)
rows = result.mappings().all()
return [dict(r) for r in rows]
@router.post("/hotels", status_code=201)
async def create_hotel(body: HotelCreate, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
if body.profile_name not in PROFILES:
raise HTTPException(status_code=400, detail=f"Unknown profile: {body.profile_name}")
async with AsyncSessionLocal() as db:
result = await db.execute(
text("""INSERT INTO direct_competitor_hotels (name, profile_name, params)
VALUES (:name, :profile, :params) RETURNING id"""),
{"name": body.name, "profile": body.profile_name, "params": json.dumps(body.params)}
)
new_id = result.fetchone()[0]
await db.commit()
return {"id": new_id, "name": body.name, "profile_name": body.profile_name}
@router.put("/hotels/{hotel_id}")
async def update_hotel(hotel_id: int, body: HotelUpdate, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
updates = {}
if body.name is not None: updates["name"] = body.name
if body.scrape_enabled is not None: updates["scrape_enabled"] = body.scrape_enabled
if body.benchmark_room is not None: updates["benchmark_room"] = body.benchmark_room
if body.benchmark_rate is not None: updates["benchmark_rate"] = body.benchmark_rate
if body.tier_base_room is not None: updates["tier_base_room"] = body.tier_base_room
if body.params is not None: updates["params"] = json.dumps(body.params)
if body.room_labels is not None: updates["room_labels"] = json.dumps(body.room_labels)
if body.rate_labels is not None: updates["rate_labels"] = json.dumps(body.rate_labels)
if body.room_order is not None: updates["room_order"] = json.dumps(body.room_order)
if body.tier_offsets is not None: updates["tier_offsets"] = json.dumps(body.tier_offsets)
if not updates:
raise HTTPException(status_code=400, detail="No fields to update")
set_clause = ", ".join(f"{k} = :{k}" for k in updates)
updates["hotel_id"] = hotel_id
async with AsyncSessionLocal() as db:
await db.execute(
text(f"UPDATE direct_competitor_hotels SET {set_clause} WHERE id = :hotel_id"),
updates
)
await db.commit()
return {"ok": True}
@router.delete("/hotels/{hotel_id}", status_code=204)
async def delete_hotel(hotel_id: int, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
async with AsyncSessionLocal() as db:
await db.execute(
text("DELETE FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
await db.commit()
# ─── Discovery ───────────────────────────────────────────────────────────────
@router.post("/hotels/{hotel_id}/discover")
async def trigger_discovery(hotel_id: int, background_tasks: BackgroundTasks, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
async with AsyncSessionLocal() as db:
row = await db.execute(
text("SELECT profile_name, params FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
background_tasks.add_task(
asyncio.get_event_loop().run_until_complete,
run_discovery(hotel_id, hotel["profile_name"], hotel["params"])
)
return {"status": "discovery started", "hotel_id": hotel_id}
@router.get("/hotels/{hotel_id}/discovery-status")
async def discovery_status(hotel_id: int, user=Depends(get_current_user)):
require_cap(user, "view_direct_rates")
return get_discovery_status(hotel_id)
# ─── Manual scrape trigger ────────────────────────────────────────────────────
@router.post("/hotels/{hotel_id}/scrape")
async def trigger_scrape(hotel_id: int, background_tasks: BackgroundTasks, user=Depends(get_current_user)):
require_cap(user, "manage_scraper")
async with AsyncSessionLocal() as db:
row = await db.execute(
text("SELECT profile_name, params FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
def _bg():
run_scrape(hotel_id, hotel["profile_name"], hotel["params"])
background_tasks.add_task(_bg)
return {"status": "scrape started", "hotel_id": hotel_id}
# ─── Rate data ────────────────────────────────────────────────────────────────
@router.get("/hotels/{hotel_id}/dates")
async def hotel_dates(
hotel_id: int,
from_date: date = None,
to_date: date = None,
user=Depends(get_current_user)
):
require_cap(user, "view_direct_rates")
if from_date is None:
from_date = date.today()
if to_date is None:
to_date = date.today() + timedelta(days=89)
async with AsyncSessionLocal() as db:
# Hotel config for labels/tier offsets
cfg_row = await db.execute(
text("SELECT name, room_labels, rate_labels, tier_offsets, benchmark_room, benchmark_rate, tier_base_room FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = cfg_row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
# Latest snapshot per date: cheapest available price_incl
rates_result = await db.execute(
text("""
SELECT
r.stay_date,
MIN(r.price_incl) FILTER (WHERE r.availability > 0) AS cheapest_rate,
BOOL_OR(r.availability > 0) AS has_availability,
BOOL_OR(r.min_stay_nights IS NOT NULL
AND r.min_stay_nights > 1) AS has_min_stay,
MAX(r.scraped_at) AS scraped_at
FROM (
SELECT DISTINCT ON (room_id, rate_id)
stay_date, room_id, rate_id, availability,
price_incl, min_stay_nights, scraped_at
FROM direct_rates
WHERE hotel_id = :hid
AND stay_date BETWEEN :fd AND :td
ORDER BY room_id, rate_id, scraped_at DESC
) r
GROUP BY r.stay_date
ORDER BY r.stay_date
"""),
{"hid": hotel_id, "fd": from_date, "td": to_date}
)
dates = [dict(r) for r in rates_result.mappings().all()]
return {
"hotel_id": hotel_id,
"hotel_name": hotel["name"],
"from_date": str(from_date),
"to_date": str(to_date),
"dates": dates,
}
@router.get("/hotels/{hotel_id}/date/{rate_date}/rooms")
async def hotel_date_rooms(
hotel_id: int,
rate_date: date,
user=Depends(get_current_user)
):
require_cap(user, "view_direct_rates")
async with AsyncSessionLocal() as db:
cfg_row = await db.execute(
text("SELECT room_labels, rate_labels, tier_offsets, tier_base_room, benchmark_room, benchmark_rate FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = cfg_row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
# Latest snapshot per room/rate for this date
rooms_result = await db.execute(
text("""
SELECT DISTINCT ON (room_id, rate_id)
room_id, rate_id, availability, price_excl, price_incl,
currency, min_stay_nights, scraped_at
FROM direct_rates
WHERE hotel_id = :hid AND stay_date = :sd
ORDER BY room_id, rate_id, scraped_at DESC
"""),
{"hid": hotel_id, "sd": rate_date}
)
rooms = [dict(r) for r in rooms_result.mappings().all()]
# Resolve tier-normalised benchmark rates
room_labels = hotel["room_labels"] or {}
rate_labels = hotel["rate_labels"] or {}
tier_offsets = hotel["tier_offsets"] or {}
tier_base_room = hotel["tier_base_room"]
benchmark_room = hotel["benchmark_room"]
benchmark_rate = hotel["benchmark_rate"]
# Find benchmark price
bench_price = None
if benchmark_room and benchmark_rate:
bench_match = next(
(r for r in rooms
if r["room_id"] == benchmark_room and r["rate_id"] == benchmark_rate
and r["availability"] > 0 and r["price_incl"]),
None
)
if bench_match:
bench_price = float(bench_match["price_incl"])
elif tier_base_room and tier_offsets:
base_match = next(
(r for r in rooms
if r["room_id"] == tier_base_room and r["availability"] > 0 and r["price_incl"]),
None
)
if base_match:
base_price = float(base_match["price_incl"])
bench_offset = tier_offsets.get(benchmark_room, 0)
base_offset = tier_offsets.get(tier_base_room, 0)
bench_price = base_price - base_offset + bench_offset
enriched = []
for r in rooms:
r["room_label"] = room_labels.get(r["room_id"], r["room_id"])
r["rate_label"] = rate_labels.get(r["rate_id"], r["rate_id"])
# Derive bench_rate for this room from tier offsets
r["bench_rate"] = None
if bench_price is not None and tier_offsets and tier_base_room:
room_offset = tier_offsets.get(r["room_id"])
bench_offset = tier_offsets.get(benchmark_room, 0)
if room_offset is not None:
r["bench_rate"] = round(bench_price + (room_offset - bench_offset), 2)
enriched.append(r)
return {
"hotel_id": hotel_id,
"date": str(rate_date),
"bench_price": bench_price,
"rooms": enriched,
}