hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add Rate Monitor app — Booking.com + direct booking engine competitor rates

Browse source 
Combines Booking.com Playwright scraper (from forecasting), direct booking
engine scraper (ported from laptop-archive/guestline-monitor), and Newbook
own-hotel rates into one focused tool. Four views: Bookability, Market View
(with price index badges + direct rate sub-rows), Direct Rates (per-competitor
room breakdown, min-stay flags, hotel config/discovery), Rate Analysis
(advance purchase curve, DOW chart, rate timeline, comparison table).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 12:06:30 +00:00
commit e05054172f
50 changed files with 11860 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
.gitignore vendored Normal file
View file

@ -0,0 +1,13 @@
node_modules/
dist/
__pycache__/
*.pyc
*.pyo
.env
.env.local
.DS_Store
*.egg-info/
.eggs/
build/
.venv/
venv/

22
backend/Dockerfile Normal file
View file

@ -0,0 +1,22 @@
FROM python:3.11-slim
WORKDIR /app
RUN apt-get update && apt-get install -y \
build-essential \
libpq-dev \
postgresql-client \
curl \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Playwright + Chromium for Booking.com scraper
RUN playwright install chromium --with-deps
COPY . .
EXPOSE 8000
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

0
backend/api/__init__.py Normal file
View file

317
backend/api/analysis.py Normal file
View file

@ -0,0 +1,317 @@
"""
Rate Analysis API — advance purchase curves, DOW analysis, rate timelines, strategy summary
"""
from fastapi import APIRouter, Depends, Query
from sqlalchemy import text
from typing import Optional
from datetime import date, timedelta
from database import AsyncSessionLocal
from auth import get_current_user, require_cap
router = APIRouter()
async def get_db():
async with AsyncSessionLocal() as db:
yield db
# ─── Hotels available for analysis ───────────────────────────────────────────
@router.get("/hotels")
async def list_analysis_hotels(
tier: Optional[str] = Query(None, description="Filter by tier: own|competitor|market"),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
async with AsyncSessionLocal() as db:
where = "is_active = true"
params = {}
if tier:
where += " AND tier = :tier"
params["tier"] = tier
result = await db.execute(
text(f"""
SELECT h.id, h.name, h.tier, h.star_rating, h.review_score,
h.booking_com_url,
COUNT(DISTINCT r.rate_date) AS scraped_dates,
MAX(r.scraped_at) AS last_scraped
FROM booking_com_hotels h
LEFT JOIN booking_com_rates r ON r.hotel_id = h.id
WHERE {where}
GROUP BY h.id, h.name, h.tier, h.star_rating, h.review_score, h.booking_com_url
ORDER BY h.tier, h.display_order, h.name
"""),
params
)
rows = result.mappings().all()
return [dict(r) for r in rows]
# ─── Full analysis for one hotel ─────────────────────────────────────────────
@router.get("/hotel/{hotel_id}")
async def analyse_hotel(
hotel_id: int,
from_date: date = Query(default_factory=lambda: date.today()),
to_date: date = Query(default_factory=lambda: date.today() + timedelta(days=89)),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
async with AsyncSessionLocal() as db:
# Hotel info
hotel_row = await db.execute(
text("SELECT id, name, tier, star_rating, booking_com_url FROM booking_com_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = hotel_row.mappings().fetchone()
if not hotel:
from fastapi import HTTPException
raise HTTPException(status_code=404, detail="Hotel not found")
# Advance purchase curve
apc_result = await db.execute(
text("""
SELECT
CASE
WHEN (rate_date - scraped_at::date) <= 7 THEN '0-7d'
WHEN (rate_date - scraped_at::date) <= 29 THEN '8-29d'
WHEN (rate_date - scraped_at::date) <= 89 THEN '30-89d'
ELSE '90+d'
END AS lead_bucket,
ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate,
COUNT(*) AS sample_count
FROM booking_com_rates
WHERE hotel_id = :hid
AND rate_date BETWEEN :from_date AND :to_date
AND availability_status = 'available'
AND rate_gross IS NOT NULL
GROUP BY lead_bucket
ORDER BY
CASE lead_bucket
WHEN '0-7d' THEN 1
WHEN '8-29d' THEN 2
WHEN '30-89d' THEN 3
ELSE 4
END
"""),
{"hid": hotel_id, "from_date": from_date, "to_date": to_date}
)
advance_purchase_curve = [dict(r) for r in apc_result.mappings().all()]
# Day-of-week averages (latest scrape per date)
dow_result = await db.execute(
text("""
SELECT
EXTRACT(DOW FROM rate_date)::int AS dow,
TO_CHAR(rate_date, 'Dy') AS dow_label,
ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate,
COUNT(DISTINCT rate_date) AS date_count
FROM (
SELECT DISTINCT ON (rate_date)
rate_date, rate_gross
FROM booking_com_rates
WHERE hotel_id = :hid
AND rate_date BETWEEN :from_date AND :to_date
AND availability_status = 'available'
AND rate_gross IS NOT NULL
ORDER BY rate_date, scraped_at DESC
) latest
GROUP BY dow, dow_label
ORDER BY dow
"""),
{"hid": hotel_id, "from_date": from_date, "to_date": to_date}
)
dow_analysis = [dict(r) for r in dow_result.mappings().all()]
# Sold-out pattern by day-of-week
sold_out_result = await db.execute(
text("""
SELECT
EXTRACT(DOW FROM rate_date)::int AS dow,
TO_CHAR(rate_date, 'Dy') AS dow_label,
COUNT(DISTINCT rate_date) AS total_dates,
COUNT(DISTINCT rate_date) FILTER (
WHERE availability_status = 'sold_out'
) AS sold_out_dates
FROM (
SELECT DISTINCT ON (rate_date)
rate_date, availability_status
FROM booking_com_rates
WHERE hotel_id = :hid
AND rate_date BETWEEN :from_date AND :to_date
ORDER BY rate_date, scraped_at DESC
) latest
GROUP BY dow, dow_label
ORDER BY dow
"""),
{"hid": hotel_id, "from_date": from_date, "to_date": to_date}
)
sold_out_pattern = [dict(r) for r in sold_out_result.mappings().all()]
# Strategy summary
strategy = _compute_strategy(advance_purchase_curve, dow_analysis, sold_out_pattern)
return {
"hotel": dict(hotel),
"date_range": {"from": str(from_date), "to": str(to_date)},
"advance_purchase_curve": advance_purchase_curve,
"dow_analysis": dow_analysis,
"sold_out_pattern": sold_out_pattern,
"strategy_summary": strategy,
}
# ─── Rate timeline for a single date ─────────────────────────────────────────
@router.get("/hotel/{hotel_id}/timeline")
async def rate_timeline(
hotel_id: int,
rate_date: date = Query(...),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
async with AsyncSessionLocal() as db:
result = await db.execute(
text("""
SELECT
scraped_at,
rate_gross,
availability_status,
rooms_left,
room_type,
(rate_date - scraped_at::date) AS days_out
FROM booking_com_rates
WHERE hotel_id = :hid AND rate_date = :rd
ORDER BY scraped_at ASC
"""),
{"hid": hotel_id, "rd": rate_date}
)
rows = result.mappings().all()
return {
"hotel_id": hotel_id,
"rate_date": str(rate_date),
"timeline": [dict(r) for r in rows],
}
# ─── Own vs Competitor comparison ────────────────────────────────────────────
@router.get("/comparison")
async def rate_comparison(
competitor_ids: str = Query(..., description="Comma-separated hotel IDs to compare"),
from_date: date = Query(default_factory=lambda: date.today()),
to_date: date = Query(default_factory=lambda: date.today() + timedelta(days=29)),
user=Depends(get_current_user)
):
require_cap(user, "rate_analysis")
try:
comp_ids = [int(x.strip()) for x in competitor_ids.split(",") if x.strip()]
except ValueError:
from fastapi import HTTPException
raise HTTPException(status_code=400, detail="competitor_ids must be comma-separated integers")
async with AsyncSessionLocal() as db:
# Own hotel latest rates
own_result = await db.execute(
text("""
SELECT
ncr.rate_date,
ncr.gross_rate AS own_rate
FROM newbook_current_rates ncr
WHERE ncr.rate_date BETWEEN :from_date AND :to_date
ORDER BY ncr.rate_date
"""),
{"from_date": from_date, "to_date": to_date}
)
own_rates = {str(r.rate_date): float(r.own_rate) for r in own_result if r.own_rate}
# Competitor latest rates per date
comp_result = await db.execute(
text("""
SELECT
r.hotel_id,
h.name AS hotel_name,
r.rate_date,
r.rate_gross,
r.availability_status
FROM (
SELECT DISTINCT ON (hotel_id, rate_date)
hotel_id, rate_date, rate_gross, availability_status
FROM booking_com_rates
WHERE hotel_id = ANY(:comp_ids)
AND rate_date BETWEEN :from_date AND :to_date
ORDER BY hotel_id, rate_date, scraped_at DESC
) r
JOIN booking_com_hotels h ON h.id = r.hotel_id
ORDER BY r.rate_date, h.display_order
"""),
{"comp_ids": comp_ids, "from_date": from_date, "to_date": to_date}
)
comp_rows = comp_result.mappings().all()
# Build per-date rows
date_map: dict = {}
hotel_names: dict = {}
for row in comp_rows:
d = str(row["rate_date"])
if d not in date_map:
date_map[d] = {"date": d, "own_rate": own_rates.get(d)}
date_map[d][f"h{row['hotel_id']}"] = float(row["rate_gross"]) if row["rate_gross"] else None
date_map[d][f"h{row['hotel_id']}_status"] = row["availability_status"]
hotel_names[row["hotel_id"]] = row["hotel_name"]
return {
"hotel_names": hotel_names,
"rows": sorted(date_map.values(), key=lambda x: x["date"]),
}
# ─── Strategy computation helper ─────────────────────────────────────────────
def _compute_strategy(apc: list, dow: list, sold_out: list) -> dict:
# Advance discount — compare 0-7d vs 30-89d
rates_by_bucket = {r["lead_bucket"]: float(r["avg_rate"]) for r in apc if r.get("avg_rate")}
advance_discount_pct = None
if "0-7d" in rates_by_bucket and "30-89d" in rates_by_bucket:
close_in = rates_by_bucket["0-7d"]
far_out = rates_by_bucket["30-89d"]
if far_out > 0:
# Positive = closes in higher (scarcity premium); negative = discount for advance
advance_discount_pct = round((close_in - far_out) / far_out * 100, 1)
# Weekend premium — Fri(5)+Sat(6) vs Mon(1)–Thu(4)
rates_by_dow = {r["dow"]: float(r["avg_rate"]) for r in dow if r.get("avg_rate")}
weekend_premium_pct = None
weekend_rates = [rates_by_dow[d] for d in [5, 6] if d in rates_by_dow]
weekday_rates = [rates_by_dow[d] for d in [1, 2, 3, 4] if d in rates_by_dow]
if weekend_rates and weekday_rates:
avg_wk = sum(weekend_rates) / len(weekend_rates)
avg_wd = sum(weekday_rates) / len(weekday_rates)
if avg_wd > 0:
weekend_premium_pct = round((avg_wk - avg_wd) / avg_wd * 100, 1)
# Sold-out rate
total_dates = sum(r["total_dates"] for r in sold_out)
sold_out_dates = sum(r["sold_out_dates"] for r in sold_out)
sold_out_rate_pct = round(sold_out_dates / total_dates * 100, 1) if total_dates > 0 else None
# Strategy label
label = "Mixed / insufficient data"
if advance_discount_pct is not None:
if advance_discount_pct <= -5:
label = "Advance-booking discounter"
elif advance_discount_pct >= 8 and (sold_out_rate_pct or 0) >= 10:
label = "Yield manager (scarcity-driven)"
elif advance_discount_pct >= 3:
label = "Flat-rate / hold-firm strategy"
else:
label = "Stable pricing"
return {
"advance_discount_pct": advance_discount_pct,
"weekend_premium_pct": weekend_premium_pct,
"avg_sold_out_rate_pct": sold_out_rate_pct,
"strategy_label": label,
}

624
backend/api/bookability.py Normal file
View file

@ -0,0 +1,624 @@
"""
Bookability API endpoints
Rate availability matrix and competitor rate comparison
"""
from typing import Optional, List, Dict, Any
from datetime import date, datetime, timedelta
from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from pydantic import BaseModel
import logging
import json
from database import get_db, SyncSessionLocal
from auth import get_current_user
router = APIRouter()
logger = logging.getLogger(__name__)
# ============================================
# RESPONSE MODELS
# ============================================
class CategoryInfo(BaseModel):
category_id: str
category_name: str
room_count: int
class TariffInfo(BaseModel):
name: str
description: Optional[str] = None
rate: Optional[float] = None
average_nightly: Optional[float] = None
available: bool
message: str
sort_order: int = 999
min_stay: Optional[int] = None
available_for_min_stay: Optional[bool] = None # True if available when queried with min_stay nights
class OccupancyInfo(BaseModel):
occupied: int = 0
available: int = 0
maintenance: int = 0
class DateRateInfo(BaseModel):
rate_gross: Optional[float] = None
rate_net: Optional[float] = None
tariffs: List[TariffInfo]
tariff_count: int
occupancy: Optional[OccupancyInfo] = None
valid_from: Optional[str] = None
class RateMatrixResponse(BaseModel):
categories: List[CategoryInfo]
dates: List[str]
matrix: Dict[str, Dict[str, DateRateInfo]]
date_last_updated: Dict[str, Optional[str]] = {}
# ============================================
# RATE MATRIX ENDPOINT
# ============================================
@router.get("/rate-matrix", response_model=RateMatrixResponse)
async def get_rate_matrix(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
category_id: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate availability matrix for all tariffs across dates and categories.
Returns a matrix showing all available tariff options for each room category
and date combination, including availability status and rates.
Args:
from_date: Start date (YYYY-MM-DD), defaults to today
to_date: End date (YYYY-MM-DD), defaults to today + 30 days
category_id: Optional filter to specific category
Returns:
RateMatrixResponse with categories, dates, and the matrix data
"""
# Default date range
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Validate date range
if end < start:
raise HTTPException(status_code=400, detail="to_date must be after from_date")
if (end - start).days > 366:
raise HTTPException(status_code=400, detail="Date range cannot exceed 366 days")
# Fetch categories
cat_query = """
SELECT site_id, site_name, room_count
FROM newbook_room_categories
WHERE is_included = true
"""
params: Dict[str, Any] = {}
if category_id:
cat_query += " AND site_id = :category_id"
params["category_id"] = category_id
cat_query += " ORDER BY display_order, site_name"
cat_result = await db.execute(text(cat_query), params)
categories = [
CategoryInfo(
category_id=row.site_id,
category_name=row.site_name,
room_count=row.room_count or 0
)
for row in cat_result.fetchall()
]
if not categories:
return RateMatrixResponse(categories=[], dates=[], matrix={})
# Build date list
dates = []
current = start
while current <= end:
dates.append(current.isoformat())
current += timedelta(days=1)
# Fetch rates with tariffs_data (get latest version per category/date)
rates_query = """
SELECT DISTINCT ON (category_id, rate_date)
category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from
FROM newbook_current_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
"""
rates_params: Dict[str, Any] = {"from_date": start, "to_date": end}
if category_id:
rates_query += " AND category_id = :category_id"
rates_params["category_id"] = category_id
rates_query += " ORDER BY category_id, rate_date, valid_from DESC"
rates_result = await db.execute(text(rates_query), rates_params)
rates_rows = rates_result.fetchall()
# Fetch occupancy data from newbook_occupancy_report_data
occupancy_query = """
SELECT category_id, date, occupied, available, maintenance
FROM newbook_occupancy_report_data
WHERE date >= :from_date AND date <= :to_date
"""
occupancy_params: Dict[str, Any] = {"from_date": start, "to_date": end}
if category_id:
occupancy_query += " AND category_id = :category_id"
occupancy_params["category_id"] = category_id
occupancy_result = await db.execute(text(occupancy_query), occupancy_params)
occupancy_rows = occupancy_result.fetchall()
# Build occupancy lookup: category_id -> date -> OccupancyInfo
occupancy_map: Dict[str, Dict[str, OccupancyInfo]] = {}
for row in occupancy_rows:
cat_id = row.category_id
occ_date = row.date.isoformat()
if cat_id not in occupancy_map:
occupancy_map[cat_id] = {}
occupancy_map[cat_id][occ_date] = OccupancyInfo(
occupied=row.occupied or 0,
available=row.available or 0,
maintenance=row.maintenance or 0
)
# Build matrix
matrix: Dict[str, Dict[str, DateRateInfo]] = {}
# Initialize matrix with empty data for all categories and dates
for cat in categories:
matrix[cat.category_id] = {}
for date_str in dates:
# Get occupancy for this category/date if available
occ = occupancy_map.get(cat.category_id, {}).get(date_str)
matrix[cat.category_id][date_str] = DateRateInfo(
rate_gross=None,
rate_net=None,
tariffs=[],
tariff_count=0,
occupancy=occ
)
# Populate matrix with actual data
for row in rates_rows:
cat_id = row.category_id
rate_date = row.rate_date.isoformat()
if cat_id not in matrix or rate_date not in matrix[cat_id]:
continue
# Parse tariffs_data
tariffs_data = row.tariffs_data or {}
if isinstance(tariffs_data, str):
try:
tariffs_data = json.loads(tariffs_data)
except json.JSONDecodeError:
tariffs_data = {}
# Build tariff list
tariffs_list = []
raw_tariffs = tariffs_data.get('tariffs', [])
for idx, tariff in enumerate(raw_tariffs):
tariffs_list.append(TariffInfo(
name=tariff.get('name', 'Unknown'),
description=tariff.get('description'),
rate=tariff.get('rate'),
average_nightly=tariff.get('average_nightly'),
available=tariff.get('success', False),
message=tariff.get('message', ''),
sort_order=tariff.get('sort_order', idx),
min_stay=tariff.get('min_stay'),
available_for_min_stay=tariff.get('available_for_min_stay')
))
# Preserve existing occupancy data
existing_occ = matrix[cat_id][rate_date].occupancy
vf = row.valid_from.isoformat() if row.valid_from else None
matrix[cat_id][rate_date] = DateRateInfo(
rate_gross=float(row.rate_gross) if row.rate_gross else None,
rate_net=float(row.rate_net) if row.rate_net else None,
tariffs=tariffs_list,
tariff_count=tariffs_data.get('tariff_count', len(tariffs_list)),
occupancy=existing_occ,
valid_from=vf
)
# Per-date latest update time (max valid_from across all categories for each date)
date_last_updated: Dict[str, Optional[str]] = {}
for date_str in dates:
latest = None
for cat in categories:
vf = matrix.get(cat.category_id, {}).get(date_str, DateRateInfo(tariffs=[], tariff_count=0)).valid_from
if vf and (latest is None or vf > latest):
latest = vf
date_last_updated[date_str] = latest
return RateMatrixResponse(
categories=categories,
dates=dates,
matrix=matrix,
date_last_updated=date_last_updated
)
# ============================================
# RATE MATRIX SUMMARY (lightweight endpoint)
# ============================================
@router.get("/rate-matrix/summary")
async def get_rate_matrix_summary(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get a summary of rate availability issues.
Returns counts of unavailable tariffs by category and date for quick
identification of potential bookability problems.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Query rates with issues (get latest version per category/date)
result = await db.execute(
text("""
SELECT DISTINCT ON (category_id, rate_date)
category_id,
rate_date,
tariffs_data
FROM newbook_current_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
AND tariffs_data IS NOT NULL
ORDER BY category_id, rate_date, valid_from DESC
"""),
{"from_date": start, "to_date": end}
)
issues = []
for row in result.fetchall():
tariffs_data = row.tariffs_data or {}
if isinstance(tariffs_data, str):
try:
tariffs_data = json.loads(tariffs_data)
except json.JSONDecodeError:
continue
tariffs = tariffs_data.get('tariffs', [])
unavailable = [t for t in tariffs if not t.get('success', False)]
if unavailable:
issues.append({
"category_id": row.category_id,
"date": row.rate_date.isoformat(),
"unavailable_count": len(unavailable),
"unavailable_tariffs": [t.get('name') for t in unavailable],
"messages": [t.get('message') for t in unavailable if t.get('message')]
})
return {
"from_date": start.isoformat(),
"to_date": end.isoformat(),
"total_issues": len(issues),
"issues": issues
}
# ============================================
# RATE HISTORY ENDPOINT
# ============================================
@router.get("/rate-history/{category_id}/{rate_date}")
async def get_rate_history(
category_id: str,
rate_date: str,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate change history for a specific category and date.
Returns all rate snapshots showing how rates evolved over time.
Useful for understanding when rates changed and by how much.
"""
try:
target_date = date.fromisoformat(rate_date)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
result = await db.execute(
text("""
SELECT
id,
rate_gross,
rate_net,
tariffs_data,
valid_from,
last_verified_at
FROM newbook_current_rates
WHERE category_id = :category_id AND rate_date = :rate_date
ORDER BY valid_from DESC
"""),
{"category_id": category_id, "rate_date": target_date}
)
history = []
for row in result.fetchall():
tariffs_data = row.tariffs_data or {}
if isinstance(tariffs_data, str):
try:
tariffs_data = json.loads(tariffs_data)
except json.JSONDecodeError:
tariffs_data = {}
tariffs = tariffs_data.get('tariffs', [])
history.append({
"id": row.id,
"rate_gross": float(row.rate_gross) if row.rate_gross else None,
"rate_net": float(row.rate_net) if row.rate_net else None,
"valid_from": row.valid_from.isoformat() if row.valid_from else None,
"last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None,
"tariff_count": len(tariffs),
"tariffs_available": sum(1 for t in tariffs if t.get('success', False)),
"tariffs_unavailable": sum(1 for t in tariffs if not t.get('success', False)),
"tariffs": [
{
"name": t.get('name'),
"rate": t.get('rate'),
"available": t.get('success', False),
"message": t.get('message', ''),
"min_stay": t.get('min_stay')
}
for t in tariffs
]
})
return {
"category_id": category_id,
"rate_date": rate_date,
"version_count": len(history),
"history": history
}
# ============================================
# RATE CHANGES SUMMARY
# ============================================
@router.get("/rate-changes")
async def get_rate_changes(
days: int = 7,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get summary of rate changes in the last N days.
Shows which rates changed and when, useful for tracking pricing strategy changes.
"""
cutoff = datetime.now() - timedelta(days=days)
# Find dates with multiple versions (indicating changes)
result = await db.execute(
text("""
SELECT
category_id,
rate_date,
COUNT(*) as version_count,
MIN(valid_from) as first_version,
MAX(valid_from) as latest_version
FROM newbook_current_rates
WHERE valid_from >= :cutoff
GROUP BY category_id, rate_date
HAVING COUNT(*) > 1
ORDER BY MAX(valid_from) DESC
LIMIT 100
"""),
{"cutoff": cutoff}
)
changes = []
for row in result.fetchall():
changes.append({
"category_id": row.category_id,
"rate_date": row.rate_date.isoformat(),
"version_count": row.version_count,
"first_version": row.first_version.isoformat() if row.first_version else None,
"latest_version": row.latest_version.isoformat() if row.latest_version else None
})
return {
"days": days,
"total_changes": len(changes),
"changes": changes
}
# ============================================
# FETCH RATES TRIGGER (manual refresh)
# ============================================
@router.post("/refresh-rates")
async def refresh_rates(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
category_id: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Trigger a manual refresh of current rates from Newbook.
This runs the rate fetch job for the specified date range.
Note: This can be slow as it respects Newbook API rate limits.
"""
from jobs.fetch_current_rates import run_fetch_current_rates
# For now, just run the standard fetch
# TODO: Add support for custom date range and category filter
try:
await run_fetch_current_rates()
return {"status": "success", "message": "Rates refresh completed"}
except Exception as e:
logger.error(f"Rates refresh failed: {e}")
raise HTTPException(status_code=500, detail=f"Rates refresh failed: {str(e)}")
# ============================================
# SINGLE-DATE RATE REFRESH
# ============================================
def _refresh_date_sync(rate_date: date):
"""
Fetch rates for a single date from Newbook and save to DB.
Runs synchronously in a background task.
"""
import asyncio
from decimal import Decimal
from services.newbook_rates_client import NewbookRatesClient
from jobs.fetch_current_rates import save_rate_snapshot
db = SyncSessionLocal()
try:
# Get config
config_result = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')
""")
)
config = {row.config_key: row.config_value for row in config_result.fetchall()}
# Central Settings service first, app-local config fallback
from services.central_settings import get_newbook_credentials_sync
creds = get_newbook_credentials_sync()
if not creds:
if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
logger.error("Newbook credentials not configured for single-date refresh")
return
creds = {
'api_key': config['newbook_api_key'],
'username': config['newbook_username'],
'password': config['newbook_password'],
'region': config['newbook_region'],
}
vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20'))
# Get included categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = set(row.site_id for row in cat_result.fetchall())
client = NewbookRatesClient(
api_key=creds['api_key'],
username=creds['username'],
password=creds['password'],
region=creds['region'],
vat_rate=vat_rate
)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
async def _fetch():
async with client:
# Single-night query for this date (all categories)
category_rates = await client._fetch_all_categories_batch(
rate_date, guests_adults=2, guests_children=0
)
# Check for min_stay tariffs needing multi-night verification
dates_by_nights: Dict[int, list] = {}
for cat_id, rates in category_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
min_stay = tariff.get('min_stay')
if min_stay and min_stay > 1 and not tariff.get('success', False):
if min_stay not in dates_by_nights:
dates_by_nights[min_stay] = []
if rate_date not in dates_by_nights[min_stay]:
dates_by_nights[min_stay].append(rate_date)
# Run multi-night verification if needed
if dates_by_nights:
multi_results = await client.get_multi_night_availability(dates_by_nights)
for cat_id, rates in category_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
if rate_date in multi_results:
cat_avail = multi_results[rate_date].get(cat_id, {})
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
if tariff.get('min_stay') and tariff['min_stay'] > 1:
tariff['available_for_min_stay'] = cat_avail.get(tariff.get('name', ''), False)
# Save snapshots
inserted = 0
for cat_id, rates in category_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
result = save_rate_snapshot(db, cat_id, rate['date'], rate)
if result == 'inserted':
inserted += 1
return inserted
inserted = loop.run_until_complete(_fetch())
db.commit()
logger.info(f"Single-date refresh for {rate_date}: {inserted} new snapshots")
finally:
loop.close()
except Exception as e:
logger.error(f"Single-date refresh failed for {rate_date}: {e}", exc_info=True)
db.rollback()
finally:
db.close()
@router.post("/refresh-date/{rate_date}")
async def refresh_single_date(
rate_date: str,
current_user: dict = Depends(get_current_user)
):
"""
Trigger a refresh of rates for a single date from Newbook.
Runs synchronously so the client can invalidate its cache immediately on response.
"""
try:
target_date = date.fromisoformat(rate_date)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
import asyncio
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, _refresh_date_sync, target_date)
return {"status": "success", "date": rate_date, "message": f"Rates refreshed for {rate_date}"}

940
backend/api/competitors.py Normal file
View file

@ -0,0 +1,940 @@
"""
Competitor Rates API endpoints
Booking.com rate scraping, hotel management, and competitor comparison
"""
from typing import Optional, List, Dict, Any
from datetime import date, datetime, timedelta
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from pydantic import BaseModel
import logging
from database import get_db, SyncSessionLocal
from auth import get_current_user
router = APIRouter()
logger = logging.getLogger(__name__)
# ============================================
# REQUEST/RESPONSE MODELS
# ============================================
class ScrapeRequest(BaseModel):
from_date: str
to_date: Optional[str] = None
class LocationConfigRequest(BaseModel):
location_name: str
pages_to_scrape: int = 2
adults: int = 2
class HotelTierUpdate(BaseModel):
tier: str # 'own', 'competitor', 'market'
display_order: Optional[int] = None
class HotelResponse(BaseModel):
id: int
booking_com_id: str
name: str
booking_com_url: Optional[str]
star_rating: Optional[float]
review_score: Optional[float]
review_count: Optional[int]
tier: str
display_order: int
notes: Optional[str]
first_seen_at: Optional[datetime]
last_seen_at: Optional[datetime]
class RateResponse(BaseModel):
rate_date: str
hotel_id: int
hotel_name: str
tier: str
star_rating: Optional[float]
review_score: Optional[float]
availability_status: str
rate_gross: Optional[float]
room_type: Optional[str]
breakfast_included: Optional[bool]
free_cancellation: Optional[bool]
no_prepayment: Optional[bool]
rooms_left: Optional[int]
scraped_at: Optional[datetime]
class ScraperStatusResponse(BaseModel):
enabled: bool
paused: bool
pause_until: Optional[str]
backend: str
location_configured: bool
location_name: Optional[str]
last_scrape: Optional[dict]
# ============================================
# SCRAPER STATUS & CONFIGURATION
# ============================================
@router.get("/status", response_model=ScraperStatusResponse)
async def get_scraper_status(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get current scraper status and configuration."""
# Get config values
config_result = await db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN (
'booking_scraper_enabled',
'booking_scraper_paused',
'booking_scraper_pause_until',
'booking_scraper_backend'
)
""")
)
config = {row.config_key: row.config_value for row in config_result.fetchall()}
# Get location config
location_result = await db.execute(
text("SELECT location_name FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
)
location_row = location_result.fetchone()
# Get last scrape info
last_scrape_result = await db.execute(
text("""
SELECT batch_id, scrape_type, started_at, completed_at, status,
hotels_found, rates_scraped, error_message
FROM booking_scrape_log
ORDER BY started_at DESC
LIMIT 1
""")
)
last_scrape_row = last_scrape_result.fetchone()
last_scrape = None
if last_scrape_row:
last_scrape = {
'batch_id': str(last_scrape_row.batch_id),
'scrape_type': last_scrape_row.scrape_type,
'started_at': last_scrape_row.started_at.isoformat() if last_scrape_row.started_at else None,
'completed_at': last_scrape_row.completed_at.isoformat() if last_scrape_row.completed_at else None,
'status': last_scrape_row.status,
'hotels_found': last_scrape_row.hotels_found,
'rates_scraped': last_scrape_row.rates_scraped,
'error_message': last_scrape_row.error_message,
}
return ScraperStatusResponse(
enabled=config.get('booking_scraper_enabled', 'false') == 'true',
paused=config.get('booking_scraper_paused', 'false') == 'true',
pause_until=config.get('booking_scraper_pause_until'),
backend=config.get('booking_scraper_backend', 'playwright_local'),
location_configured=location_row is not None,
location_name=location_row.location_name if location_row else None,
last_scrape=last_scrape
)
@router.post("/config/location")
async def set_location_config(
config: LocationConfigRequest,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Set the location to scrape for competitor rates."""
# Deactivate existing configs
await db.execute(
text("UPDATE booking_scrape_config SET is_active = FALSE")
)
# Insert new config
await db.execute(
text("""
INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, is_active)
VALUES (:location, :pages, :adults, TRUE)
"""),
{'location': config.location_name, 'pages': config.pages_to_scrape, 'adults': config.adults}
)
await db.commit()
return {"status": "success", "location": config.location_name}
@router.post("/config/enable")
async def enable_scraper(
enabled: bool = True,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Enable or disable the booking.com scraper."""
await db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_enabled'"),
{'val': 'true' if enabled else 'false'}
)
await db.commit()
return {"status": "success", "enabled": enabled}
@router.post("/config/unpause")
async def unpause_scraper(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Manually unpause the scraper (clears blocking pause)."""
await db.execute(
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
)
await db.commit()
return {"status": "success", "message": "Scraper unpaused"}
# ============================================
# MANUAL SCRAPE TRIGGER
# ============================================
def run_scrape_sync(from_date: date, to_date: date):
"""Run scrape in sync context for background task."""
import asyncio
from services.booking_scraper import run_manual_scrape, cleanup_stale_batches
db = SyncSessionLocal()
try:
# Clean up any stale batches before starting
cleanup_stale_batches(db, max_age_minutes=60)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
result = loop.run_until_complete(run_manual_scrape(db, from_date, to_date))
logger.info(f"Background scrape completed: {result}")
finally:
loop.close()
except Exception as e:
logger.error(f"Background scrape failed: {e}", exc_info=True)
finally:
db.close()
@router.post("/scrape")
async def trigger_manual_scrape(
request: ScrapeRequest,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Trigger a manual scrape for the specified date range.
Runs in background - check /status for progress.
"""
try:
from_date = date.fromisoformat(request.from_date)
to_date = date.fromisoformat(request.to_date) if request.to_date else from_date
except ValueError as e:
raise HTTPException(status_code=400, detail=f"Invalid date format: {e}")
if to_date < from_date:
raise HTTPException(status_code=400, detail="to_date must be after from_date")
if (to_date - from_date).days > 30:
raise HTTPException(status_code=400, detail="Date range cannot exceed 30 days for manual scrape")
# Check if location is configured
location_result = await db.execute(
text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
)
if not location_result.fetchone():
raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
# Check if paused
paused_result = await db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
)
paused_row = paused_result.fetchone()
if paused_row and paused_row.config_value == 'true':
raise HTTPException(status_code=400, detail="Scraper is currently paused. Use /unpause first or wait for cooldown.")
# Start background task
background_tasks.add_task(run_scrape_sync, from_date, to_date)
return {
"status": "started",
"from_date": from_date.isoformat(),
"to_date": to_date.isoformat(),
"message": "Scrape started in background. Check /status for progress."
}
# ============================================
# HOTELS MANAGEMENT
# ============================================
@router.get("/hotels", response_model=List[HotelResponse])
async def list_hotels(
tier: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
List all discovered hotels.
Filter by tier: 'own', 'competitor', 'market', or None for all.
"""
query = """
SELECT id, booking_com_id, name, booking_com_url,
star_rating, review_score, review_count,
tier, display_order, notes, first_seen_at, last_seen_at
FROM booking_com_hotels
WHERE is_active = TRUE
"""
params = {}
if tier:
if tier not in ('own', 'competitor', 'market'):
raise HTTPException(status_code=400, detail="Invalid tier. Must be 'own', 'competitor', or 'market'")
query += " AND tier = :tier"
params['tier'] = tier
query += " ORDER BY display_order, name"
result = await db.execute(text(query), params)
return [
HotelResponse(
id=row.id,
booking_com_id=row.booking_com_id or '',
name=row.name,
booking_com_url=row.booking_com_url,
star_rating=float(row.star_rating) if row.star_rating else None,
review_score=float(row.review_score) if row.review_score else None,
review_count=row.review_count,
tier=row.tier,
display_order=row.display_order,
notes=row.notes,
first_seen_at=row.first_seen_at,
last_seen_at=row.last_seen_at
)
for row in result.fetchall()
]
@router.put("/hotels/{hotel_id}/tier")
async def update_hotel_tier(
hotel_id: int,
update: HotelTierUpdate,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Update a hotel's tier and display order.
Tiers:
- 'own': Your hotel (for parity checking)
- 'competitor': Main competitors (full tracking)
- 'market': Other hotels (context only)
"""
if update.tier not in ('own', 'competitor', 'market'):
raise HTTPException(status_code=400, detail="Invalid tier")
# If setting as 'own', clear any existing 'own' hotel
if update.tier == 'own':
await db.execute(
text("UPDATE booking_com_hotels SET tier = 'market' WHERE tier = 'own'")
)
# Update the hotel
set_clause = "tier = :tier"
params = {'hotel_id': hotel_id, 'tier': update.tier}
if update.display_order is not None:
set_clause += ", display_order = :order"
params['order'] = update.display_order
result = await db.execute(
text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id RETURNING id"),
params
)
if not result.fetchone():
raise HTTPException(status_code=404, detail="Hotel not found")
await db.commit()
# If this is now the own hotel, update system config
if update.tier == 'own':
await db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_own_hotel_id'"),
{'val': str(hotel_id)}
)
await db.commit()
return {"status": "success", "hotel_id": hotel_id, "tier": update.tier}
@router.put("/hotels/{hotel_id}/notes")
async def update_hotel_notes(
hotel_id: int,
notes: str,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Update notes for a hotel."""
result = await db.execute(
text("UPDATE booking_com_hotels SET notes = :notes WHERE id = :hotel_id RETURNING id"),
{'hotel_id': hotel_id, 'notes': notes}
)
if not result.fetchone():
raise HTTPException(status_code=404, detail="Hotel not found")
await db.commit()
return {"status": "success"}
# ============================================
# COMPETITOR RATES MATRIX
# ============================================
@router.get("/matrix")
async def get_competitor_matrix(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
include_market: bool = False,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate comparison matrix for competitors.
Returns rates for own hotel and competitors, organized by date.
Set include_market=true to also include market tier hotels.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
if end < start:
raise HTTPException(status_code=400, detail="to_date must be after from_date")
if (end - start).days > 90:
raise HTTPException(status_code=400, detail="Date range cannot exceed 90 days")
tier_filter = "h.tier IN ('own', 'competitor')"
if include_market:
tier_filter = "h.tier IN ('own', 'competitor', 'market')"
# Get hotels
hotels_result = await db.execute(
text(f"""
SELECT id, name, tier, display_order, star_rating, review_score, booking_com_url
FROM booking_com_hotels
WHERE is_active = TRUE AND {tier_filter.replace('h.', '')}
ORDER BY display_order, name
""")
)
hotels = [dict(row._mapping) for row in hotels_result.fetchall()]
# Get latest rates using the view
rates_result = await db.execute(
text(f"""
SELECT DISTINCT ON (r.hotel_id, r.rate_date)
r.hotel_id,
r.rate_date,
r.availability_status,
r.rate_gross,
r.room_type,
r.breakfast_included,
r.free_cancellation,
r.no_prepayment,
r.rooms_left,
r.scraped_at
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE {tier_filter}
AND h.is_active = TRUE
AND r.rate_date >= :from_date AND r.rate_date <= :to_date
ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC
"""),
{'from_date': start, 'to_date': end}
)
# Build matrix: hotel_id -> date -> rate data
rates_by_hotel: Dict[int, Dict[str, dict]] = {}
for row in rates_result.fetchall():
hotel_id = row.hotel_id
rate_date = row.rate_date.isoformat()
if hotel_id not in rates_by_hotel:
rates_by_hotel[hotel_id] = {}
rates_by_hotel[hotel_id][rate_date] = {
'availability_status': row.availability_status,
'rate_gross': float(row.rate_gross) if row.rate_gross else None,
'room_type': row.room_type,
'breakfast_included': row.breakfast_included,
'free_cancellation': row.free_cancellation,
'no_prepayment': row.no_prepayment,
'rooms_left': row.rooms_left,
'scraped_at': row.scraped_at.isoformat() if row.scraped_at else None,
}
# Build date list
dates = []
current = start
while current <= end:
dates.append(current.isoformat())
current += timedelta(days=1)
return {
'from_date': start.isoformat(),
'to_date': end.isoformat(),
'dates': dates,
'hotels': hotels,
'rates': rates_by_hotel
}
# ============================================
# RATE PARITY (OWN HOTEL VS NEWBOOK)
# ============================================
@router.get("/parity")
async def get_rate_parity(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get rate parity comparison between booking.com and Newbook rates.
Compares scraped booking.com rates for own hotel against Newbook current rates.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Get own hotel's booking.com rates
booking_rates_result = await db.execute(
text("""
SELECT DISTINCT ON (r.rate_date)
r.rate_date,
r.rate_gross as booking_rate,
r.availability_status,
r.room_type as booking_room_type,
r.scraped_at
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE h.tier = 'own'
AND r.rate_date >= :from_date AND r.rate_date <= :to_date
ORDER BY r.rate_date, r.scraped_at DESC
"""),
{'from_date': start, 'to_date': end}
)
booking_rates = {row.rate_date: dict(row._mapping) for row in booking_rates_result.fetchall()}
# Get Newbook rates (best rate per date across categories)
newbook_rates_result = await db.execute(
text("""
SELECT DISTINCT ON (rate_date)
rate_date,
rate_gross as newbook_rate,
category_id
FROM newbook_current_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
ORDER BY rate_date, valid_from DESC
"""),
{'from_date': start, 'to_date': end}
)
newbook_rates = {row.rate_date: dict(row._mapping) for row in newbook_rates_result.fetchall()}
# Compare rates
parity_issues = []
all_dates = set(booking_rates.keys()) | set(newbook_rates.keys())
for rate_date in sorted(all_dates):
booking = booking_rates.get(rate_date)
newbook = newbook_rates.get(rate_date)
if not booking or not newbook:
continue
booking_rate = booking.get('booking_rate')
newbook_rate = newbook.get('newbook_rate')
if not booking_rate or not newbook_rate:
continue
diff_pct = ((float(booking_rate) - float(newbook_rate)) / float(newbook_rate)) * 100
if abs(diff_pct) > 1: # More than 1% difference
parity_issues.append({
'rate_date': rate_date.isoformat(),
'booking_rate': float(booking_rate),
'newbook_rate': float(newbook_rate),
'difference_pct': round(diff_pct, 2),
'alert_type': 'higher' if diff_pct > 0 else 'lower',
'booking_room_type': booking.get('booking_room_type'),
'availability_status': booking.get('availability_status'),
})
return {
'from_date': start.isoformat(),
'to_date': end.isoformat(),
'issues_count': len(parity_issues),
'issues': parity_issues
}
# ============================================
# PARITY ALERTS
# ============================================
@router.get("/parity/alerts")
async def get_parity_alerts(
status: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get rate parity alerts."""
query = """
SELECT id, rate_date, room_category, newbook_rate, booking_com_rate,
difference_pct, alert_type, alert_status, created_at,
acknowledged_at, acknowledged_by, notes
FROM rate_parity_alerts
"""
params = {}
if status:
query += " WHERE alert_status = :status"
params['status'] = status
query += " ORDER BY rate_date DESC, created_at DESC LIMIT 100"
result = await db.execute(text(query), params)
return [
{
'id': row.id,
'rate_date': row.rate_date.isoformat(),
'room_category': row.room_category,
'newbook_rate': float(row.newbook_rate) if row.newbook_rate else None,
'booking_com_rate': float(row.booking_com_rate) if row.booking_com_rate else None,
'difference_pct': float(row.difference_pct) if row.difference_pct else None,
'alert_type': row.alert_type,
'alert_status': row.alert_status,
'created_at': row.created_at.isoformat() if row.created_at else None,
'acknowledged_at': row.acknowledged_at.isoformat() if row.acknowledged_at else None,
'acknowledged_by': row.acknowledged_by,
'notes': row.notes,
}
for row in result.fetchall()
]
@router.put("/parity/alerts/{alert_id}/acknowledge")
async def acknowledge_parity_alert(
alert_id: int,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Acknowledge a parity alert."""
result = await db.execute(
text("""
UPDATE rate_parity_alerts
SET alert_status = 'acknowledged',
acknowledged_at = NOW(),
acknowledged_by = :username
WHERE id = :alert_id
RETURNING id
"""),
{'alert_id': alert_id, 'username': current_user.get('username', 'unknown')}
)
if not result.fetchone():
raise HTTPException(status_code=404, detail="Alert not found")
await db.commit()
return {"status": "success"}
# ============================================
# QUEUE STATUS
# ============================================
@router.get("/queue-status")
async def get_queue_status(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get current scrape queue status."""
result = await db.execute(
text("""
SELECT
status,
COUNT(*) as count,
MIN(rate_date) as earliest_date,
MAX(rate_date) as latest_date
FROM booking_scrape_queue
GROUP BY status
""")
)
status_counts = {row.status: {
'count': row.count,
'earliest': row.earliest_date.isoformat() if row.earliest_date else None,
'latest': row.latest_date.isoformat() if row.latest_date else None,
} for row in result.fetchall()}
# Get retry items (failed but under max_attempts)
retry_result = await db.execute(
text("""
SELECT COUNT(*) as count
FROM booking_scrape_queue
WHERE status = 'pending' AND attempts > 0
""")
)
retry_count = retry_result.fetchone().count
return {
'statuses': status_counts,
'retries_pending': retry_count,
'total_pending': status_counts.get('pending', {}).get('count', 0),
'total_completed': status_counts.get('completed', {}).get('count', 0),
'total_failed': status_counts.get('failed', {}).get('count', 0),
}
# ============================================
# SCHEDULE INFO
# ============================================
@router.get("/schedule-info")
async def get_schedule_info(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get information about the scraping schedule."""
# Get configured time
time_result = await db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_daily_time'")
)
time_row = time_result.fetchone()
daily_time = time_row.config_value if time_row and time_row.config_value else '05:30'
# Calculate what today's schedule would look like
from jobs.scrape_booking_rates import get_high_priority_dates, get_medium_priority_dates, get_low_priority_dates
high = get_high_priority_dates()
medium = get_medium_priority_dates()
low = get_low_priority_dates()
today = date.today()
weekday_name = today.strftime('%A')
return {
'daily_time': daily_time,
'today': today.isoformat(),
'weekday': weekday_name,
'tiers': {
'high': {
'description': 'Next 30 days (scraped first)',
'dates_today': len(high),
'range': f'{high[0].isoformat()} to {high[-1].isoformat()}' if high else None,
},
'medium': {
'description': 'Days 31-180 (scraped after high priority)',
'dates_today': len(medium),
'range': f'{medium[0].isoformat()} to {medium[-1].isoformat()}' if medium else None,
},
'low': {
'description': 'Days 181-365 (scraped last, or until rate limit)',
'dates_today': len(low),
'range': f'{low[0].isoformat()} to {low[-1].isoformat()}' if low else None,
},
},
'total_dates_today': len(set(high + medium + low)),
}
# ============================================
# SCRAPE COVERAGE (365-day view)
# ============================================
@router.get("/scrape-coverage")
async def get_scrape_coverage(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Get 365-day scrape coverage showing last scraped time
and next expected scrape for every date.
"""
today = date.today()
end = today + timedelta(days=365)
# Get latest scraped_at per date (across all hotels)
result = await db.execute(
text("""
SELECT rate_date, MAX(scraped_at) as last_scraped
FROM booking_com_rates
WHERE rate_date >= :from_date AND rate_date <= :to_date
GROUP BY rate_date
"""),
{'from_date': today, 'to_date': end}
)
scraped_map = {row.rate_date: row.last_scraped for row in result.fetchall()}
# Compute tier and next scrape for each date
from jobs.scrape_booking_rates import compute_next_scrape_for_date
coverage = []
for offset in range(366):
d = today + timedelta(days=offset)
tier, next_scrape = compute_next_scrape_for_date(d)
last_scraped = scraped_map.get(d)
coverage.append({
'date': d.isoformat(),
'tier': tier,
'last_scraped': last_scraped.isoformat() if last_scraped else None,
'next_expected': next_scrape.isoformat() if next_scrape else None,
})
return {
'today': today.isoformat(),
'coverage': coverage,
}
# ============================================
# BOOKING.COM AVAILABILITY CHECK (for Bookability page)
# ============================================
@router.get("/booking-availability")
async def get_booking_availability(
from_date: Optional[str] = None,
to_date: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""
Check own hotel's availability on booking.com.
Returns a simple summary: for each date in the range, whether the own hotel
appears available on booking.com based on the latest scrape data.
"""
today = date.today()
start = date.fromisoformat(from_date) if from_date else today
end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
# Get own hotel's latest scraped availability
result = await db.execute(
text("""
SELECT DISTINCT ON (r.rate_date)
r.rate_date,
r.availability_status,
r.rate_gross,
r.scraped_at
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE h.tier = 'own'
AND r.rate_date >= :from_date AND r.rate_date <= :to_date
ORDER BY r.rate_date, r.scraped_at DESC
"""),
{'from_date': start, 'to_date': end}
)
rows = result.fetchall()
if not rows:
return {
'has_own_hotel': False,
'dates_checked': 0,
'dates_available': 0,
'dates_sold_out': 0,
'dates_no_data': 0,
'latest_scrape': None,
'dates': {},
}
dates_map = {}
dates_available = 0
dates_sold_out = 0
dates_no_data = 0
latest_scrape = None
for row in rows:
status = row.availability_status
dates_map[row.rate_date.isoformat()] = {
'status': status,
'rate': float(row.rate_gross) if row.rate_gross else None,
}
if status == 'available':
dates_available += 1
elif status == 'sold_out':
dates_sold_out += 1
else:
dates_no_data += 1
if row.scraped_at and (not latest_scrape or row.scraped_at > latest_scrape):
latest_scrape = row.scraped_at
return {
'has_own_hotel': True,
'dates_checked': len(rows),
'dates_available': dates_available,
'dates_sold_out': dates_sold_out,
'dates_no_data': dates_no_data,
'latest_scrape': latest_scrape.isoformat() if latest_scrape else None,
'dates': dates_map,
}
# ============================================
# SCRAPE HISTORY
# ============================================
@router.get("/scrape-history")
async def get_scrape_history(
limit: int = 20,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user)
):
"""Get recent scrape batch history."""
result = await db.execute(
text("""
SELECT batch_id, scrape_type, started_at, completed_at, status,
dates_queued, dates_completed, dates_failed,
hotels_found, rates_scraped, error_message,
blocked_at, resume_after
FROM booking_scrape_log
ORDER BY started_at DESC
LIMIT :limit
"""),
{'limit': limit}
)
return [
{
'batch_id': str(row.batch_id),
'scrape_type': row.scrape_type,
'started_at': row.started_at.isoformat() if row.started_at else None,
'completed_at': row.completed_at.isoformat() if row.completed_at else None,
'status': row.status,
'dates_queued': row.dates_queued,
'dates_completed': row.dates_completed,
'dates_failed': row.dates_failed,
'hotels_found': row.hotels_found,
'rates_scraped': row.rates_scraped,
'error_message': row.error_message,
'blocked_at': row.blocked_at.isoformat() if row.blocked_at else None,
'resume_after': row.resume_after.isoformat() if row.resume_after else None,
}
for row in result.fetchall()
]

337
backend/api/direct.py Normal file
View file

@ -0,0 +1,337 @@
"""
Direct booking engine API — hotel management, discovery, scrape control, rate data.
"""
import asyncio
import json
import logging
from datetime import date, timedelta
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from pydantic import BaseModel
from sqlalchemy import text
from database import AsyncSessionLocal, SyncSessionLocal
from auth import get_current_user, require_cap
from services.direct_profiles import PROFILES, detect_profile, get_profile
from services.direct_scraper import run_discovery, get_discovery_status, run_scrape
router = APIRouter()
log = logging.getLogger(__name__)
# ─── Pydantic models ─────────────────────────────────────────────────────────
class HotelCreate(BaseModel):
name: str
profile_name: str
params: dict
class HotelUpdate(BaseModel):
name: Optional[str] = None
room_labels: Optional[dict] = None
rate_labels: Optional[dict] = None
room_order: Optional[list] = None
benchmark_room: Optional[str] = None
benchmark_rate: Optional[str] = None
tier_base_room: Optional[str] = None
tier_offsets: Optional[dict] = None
scrape_enabled: Optional[bool] = None
params: Optional[dict] = None
class DetectRequest(BaseModel):
url: str
# ─── Profiles ────────────────────────────────────────────────────────────────
@router.get("/profiles")
async def list_profiles(user=Depends(get_current_user)):
require_cap(user, "view_direct_rates")
return [
{
"name": name,
"label": cls.label,
"required_params": cls.required_params,
}
for name, cls in PROFILES.items()
]
@router.post("/profiles/detect")
async def detect_engine(req: DetectRequest, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
result = detect_profile(req.url)
if not result:
raise HTTPException(status_code=422, detail="Could not detect booking engine from URL")
return result
# ─── Hotel CRUD ───────────────────────────────────────────────────────────────
@router.get("/hotels")
async def list_hotels(user=Depends(get_current_user)):
require_cap(user, "view_direct_rates")
async with AsyncSessionLocal() as db:
result = await db.execute(
text("""SELECT h.id, h.name, h.profile_name, h.params,
h.room_labels, h.rate_labels, h.room_order,
h.benchmark_room, h.benchmark_rate, h.tier_base_room, h.tier_offsets,
h.scrape_enabled, h.last_scraped_at,
COUNT(DISTINCT r.stay_date) AS scraped_dates,
MAX(r.scraped_at) AS last_rate_at
FROM direct_competitor_hotels h
LEFT JOIN direct_rates r ON r.hotel_id = h.id
GROUP BY h.id
ORDER BY h.id""")
)
rows = result.mappings().all()
return [dict(r) for r in rows]
@router.post("/hotels", status_code=201)
async def create_hotel(body: HotelCreate, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
if body.profile_name not in PROFILES:
raise HTTPException(status_code=400, detail=f"Unknown profile: {body.profile_name}")
async with AsyncSessionLocal() as db:
result = await db.execute(
text("""INSERT INTO direct_competitor_hotels (name, profile_name, params)
VALUES (:name, :profile, :params) RETURNING id"""),
{"name": body.name, "profile": body.profile_name, "params": json.dumps(body.params)}
)
new_id = result.fetchone()[0]
await db.commit()
return {"id": new_id, "name": body.name, "profile_name": body.profile_name}
@router.put("/hotels/{hotel_id}")
async def update_hotel(hotel_id: int, body: HotelUpdate, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
updates = {}
if body.name is not None: updates["name"] = body.name
if body.scrape_enabled is not None: updates["scrape_enabled"] = body.scrape_enabled
if body.benchmark_room is not None: updates["benchmark_room"] = body.benchmark_room
if body.benchmark_rate is not None: updates["benchmark_rate"] = body.benchmark_rate
if body.tier_base_room is not None: updates["tier_base_room"] = body.tier_base_room
if body.params is not None: updates["params"] = json.dumps(body.params)
if body.room_labels is not None: updates["room_labels"] = json.dumps(body.room_labels)
if body.rate_labels is not None: updates["rate_labels"] = json.dumps(body.rate_labels)
if body.room_order is not None: updates["room_order"] = json.dumps(body.room_order)
if body.tier_offsets is not None: updates["tier_offsets"] = json.dumps(body.tier_offsets)
if not updates:
raise HTTPException(status_code=400, detail="No fields to update")
set_clause = ", ".join(f"{k} = :{k}" for k in updates)
updates["hotel_id"] = hotel_id
async with AsyncSessionLocal() as db:
await db.execute(
text(f"UPDATE direct_competitor_hotels SET {set_clause} WHERE id = :hotel_id"),
updates
)
await db.commit()
return {"ok": True}
@router.delete("/hotels/{hotel_id}", status_code=204)
async def delete_hotel(hotel_id: int, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
async with AsyncSessionLocal() as db:
await db.execute(
text("DELETE FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
await db.commit()
# ─── Discovery ───────────────────────────────────────────────────────────────
@router.post("/hotels/{hotel_id}/discover")
async def trigger_discovery(hotel_id: int, background_tasks: BackgroundTasks, user=Depends(get_current_user)):
require_cap(user, "manage_hotels")
async with AsyncSessionLocal() as db:
row = await db.execute(
text("SELECT profile_name, params FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
background_tasks.add_task(
asyncio.get_event_loop().run_until_complete,
run_discovery(hotel_id, hotel["profile_name"], hotel["params"])
)
return {"status": "discovery started", "hotel_id": hotel_id}
@router.get("/hotels/{hotel_id}/discovery-status")
async def discovery_status(hotel_id: int, user=Depends(get_current_user)):
require_cap(user, "view_direct_rates")
return get_discovery_status(hotel_id)
# ─── Manual scrape trigger ────────────────────────────────────────────────────
@router.post("/hotels/{hotel_id}/scrape")
async def trigger_scrape(hotel_id: int, background_tasks: BackgroundTasks, user=Depends(get_current_user)):
require_cap(user, "manage_scraper")
async with AsyncSessionLocal() as db:
row = await db.execute(
text("SELECT profile_name, params FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
def _bg():
run_scrape(hotel_id, hotel["profile_name"], hotel["params"])
background_tasks.add_task(_bg)
return {"status": "scrape started", "hotel_id": hotel_id}
# ─── Rate data ────────────────────────────────────────────────────────────────
@router.get("/hotels/{hotel_id}/dates")
async def hotel_dates(
hotel_id: int,
from_date: date = None,
to_date: date = None,
user=Depends(get_current_user)
):
require_cap(user, "view_direct_rates")
if from_date is None:
from_date = date.today()
if to_date is None:
to_date = date.today() + timedelta(days=89)
async with AsyncSessionLocal() as db:
# Hotel config for labels/tier offsets
cfg_row = await db.execute(
text("SELECT name, room_labels, rate_labels, tier_offsets, benchmark_room, benchmark_rate, tier_base_room FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = cfg_row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
# Latest snapshot per date: cheapest available price_incl
rates_result = await db.execute(
text("""
SELECT
r.stay_date,
MIN(r.price_incl) FILTER (WHERE r.availability > 0) AS cheapest_rate,
BOOL_OR(r.availability > 0) AS has_availability,
BOOL_OR(r.min_stay_nights IS NOT NULL
AND r.min_stay_nights > 1) AS has_min_stay,
MAX(r.scraped_at) AS scraped_at
FROM (
SELECT DISTINCT ON (room_id, rate_id)
stay_date, room_id, rate_id, availability,
price_incl, min_stay_nights, scraped_at
FROM direct_rates
WHERE hotel_id = :hid
AND stay_date BETWEEN :fd AND :td
ORDER BY room_id, rate_id, scraped_at DESC
) r
GROUP BY r.stay_date
ORDER BY r.stay_date
"""),
{"hid": hotel_id, "fd": from_date, "td": to_date}
)
dates = [dict(r) for r in rates_result.mappings().all()]
return {
"hotel_id": hotel_id,
"hotel_name": hotel["name"],
"from_date": str(from_date),
"to_date": str(to_date),
"dates": dates,
}
@router.get("/hotels/{hotel_id}/date/{rate_date}/rooms")
async def hotel_date_rooms(
hotel_id: int,
rate_date: date,
user=Depends(get_current_user)
):
require_cap(user, "view_direct_rates")
async with AsyncSessionLocal() as db:
cfg_row = await db.execute(
text("SELECT room_labels, rate_labels, tier_offsets, tier_base_room, benchmark_room, benchmark_rate FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
)
hotel = cfg_row.mappings().fetchone()
if not hotel:
raise HTTPException(status_code=404, detail="Hotel not found")
# Latest snapshot per room/rate for this date
rooms_result = await db.execute(
text("""
SELECT DISTINCT ON (room_id, rate_id)
room_id, rate_id, availability, price_excl, price_incl,
currency, min_stay_nights, scraped_at
FROM direct_rates
WHERE hotel_id = :hid AND stay_date = :sd
ORDER BY room_id, rate_id, scraped_at DESC
"""),
{"hid": hotel_id, "sd": rate_date}
)
rooms = [dict(r) for r in rooms_result.mappings().all()]
# Resolve tier-normalised benchmark rates
room_labels = hotel["room_labels"] or {}
rate_labels = hotel["rate_labels"] or {}
tier_offsets = hotel["tier_offsets"] or {}
tier_base_room = hotel["tier_base_room"]
benchmark_room = hotel["benchmark_room"]
benchmark_rate = hotel["benchmark_rate"]
# Find benchmark price
bench_price = None
if benchmark_room and benchmark_rate:
bench_match = next(
(r for r in rooms
if r["room_id"] == benchmark_room and r["rate_id"] == benchmark_rate
and r["availability"] > 0 and r["price_incl"]),
None
)
if bench_match:
bench_price = float(bench_match["price_incl"])
elif tier_base_room and tier_offsets:
base_match = next(
(r for r in rooms
if r["room_id"] == tier_base_room and r["availability"] > 0 and r["price_incl"]),
None
)
if base_match:
base_price = float(base_match["price_incl"])
bench_offset = tier_offsets.get(benchmark_room, 0)
base_offset = tier_offsets.get(tier_base_room, 0)
bench_price = base_price - base_offset + bench_offset
enriched = []
for r in rooms:
r["room_label"] = room_labels.get(r["room_id"], r["room_id"])
r["rate_label"] = rate_labels.get(r["rate_id"], r["rate_id"])
# Derive bench_rate for this room from tier offsets
r["bench_rate"] = None
if bench_price is not None and tier_offsets and tier_base_room:
room_offset = tier_offsets.get(r["room_id"])
bench_offset = tier_offsets.get(benchmark_room, 0)
if room_offset is not None:
r["bench_rate"] = round(bench_price + (room_offset - bench_offset), 2)
enriched.append(r)
return {
"hotel_id": hotel_id,
"date": str(rate_date),
"bench_price": bench_price,
"rooms": enriched,
}

69
backend/auth.py Normal file
View file

@ -0,0 +1,69 @@
"""
Auth middleware — verifies the stack's hnf_session cookie using the shared
CENTRAL_AUTH_SECRET.
"""
import os
from typing import Optional
from fastapi import Depends, HTTPException, Request, status
from jose import JWTError, jwt
CENTRAL_AUTH_SECRET = os.getenv("CENTRAL_AUTH_SECRET", "")
JWT_ALGORITHM = "HS256"
APP_SLUG = os.getenv("APP_SLUG", "rates")
async def get_current_user(request: Request) -> dict:
token = request.cookies.get("hnf_session")
if not token:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
try:
payload = jwt.decode(token, CENTRAL_AUTH_SECRET, algorithms=[JWT_ALGORITHM])
except JWTError:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session")
apps = payload.get("apps", [])
if APP_SLUG not in apps:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No permission for this app")
prefix = f"{APP_SLUG}:"
raw_caps = payload.get("caps", [])
if isinstance(raw_caps, list):
caps = [c[len(prefix):] for c in raw_caps if c.startswith(prefix)]
else:
caps = []
is_admin = payload.get("is_admin", False)
return {
"id": 0,
"username": payload.get("sub", ""),
"email": payload.get("sub", ""),
"display_name": payload.get("name", ""),
"name": payload.get("name", ""),
"is_admin": is_admin,
"caps": caps,
"role": "admin" if is_admin else "user",
}
def has_cap(user: dict, cap: str) -> bool:
return user.get("is_admin", False) or cap in user.get("caps", [])
def require_cap(cap: str):
async def checker(user: dict = Depends(get_current_user)):
if not has_cap(user, cap):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"Missing capability: {cap}",
)
return user
return checker
async def get_admin_user(user: dict = Depends(get_current_user)) -> dict:
if not user.get("is_admin", False):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user

37
backend/database.py Normal file
View file

@ -0,0 +1,37 @@
"""
Database connection and session management
"""
import os
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker, declarative_base
from sqlalchemy import create_engine
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://rates:rates_secret@localhost:5432/rates_db")
ASYNC_DATABASE_URL = DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://")
async_engine = create_async_engine(ASYNC_DATABASE_URL, echo=False)
AsyncSessionLocal = sessionmaker(
async_engine, class_=AsyncSession, expire_on_commit=False
)
sync_engine = create_engine(DATABASE_URL)
SyncSessionLocal = sessionmaker(bind=sync_engine)
Base = declarative_base()
async def get_db():
async with AsyncSessionLocal() as session:
try:
yield session
finally:
await session.close()
def get_sync_db():
db = SyncSessionLocal()
try:
yield db
finally:
db.close()

0
backend/jobs/__init__.py Normal file
View file

370
backend/jobs/fetch_current_rates.py Normal file
View file

@ -0,0 +1,370 @@
"""
Fetch Current Rates Job
Fetches current rack rates from Newbook API and populates newbook_current_rates table.
These rates are used by pickup-v2 model for upper bound calculations in confidence shading.
Uses a snapshot model - only inserts new rows when rates change, otherwise updates last_verified_at.
This allows tracking rate history over time.
Schedule: Daily at 5:20 AM (before pace snapshot runs)
Processing: Day-by-day with progressive DB commits. Each date is fully processed
(single-night fetch + inline multi-night verification) and saved before moving to the next.
If the job fails partway, all previously processed dates are preserved.
"""
import json
import logging
import re
from datetime import date, timedelta
from decimal import Decimal
from typing import Dict, Any, Optional, Set
import asyncio
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
COMMIT_BATCH_SIZE = 10 # Commit to DB every N days
def rates_changed(old_rate: Optional[Dict], new_rate: Dict) -> bool:
"""
Compare old and new rates to determine if they've changed.
Compares gross rate, net rate, and tariff availability status.
Returns True if rates have changed, False if they're the same.
"""
if old_rate is None:
return True # No existing rate, need to insert
# Compare gross and net rates
old_gross = float(old_rate.get('rate_gross') or 0)
new_gross = float(new_rate.get('gross_rate') or 0)
if abs(old_gross - new_gross) > 0.01:
return True
old_net = float(old_rate.get('rate_net') or 0)
new_net = float(new_rate.get('net_rate') or 0)
if abs(old_net - new_net) > 0.01:
return True
# Compare tariff availability
old_tariffs = old_rate.get('tariffs_data', {})
if isinstance(old_tariffs, str):
try:
old_tariffs = json.loads(old_tariffs)
except json.JSONDecodeError:
old_tariffs = {}
new_tariffs = new_rate.get('tariffs_data', {})
old_tariff_list = old_tariffs.get('tariffs', [])
new_tariff_list = new_tariffs.get('tariffs', [])
# Different number of tariffs
if len(old_tariff_list) != len(new_tariff_list):
return True
# Compare each tariff's key attributes
for old_t, new_t in zip(old_tariff_list, new_tariff_list):
# Name changed
if old_t.get('name') != new_t.get('name'):
return True
# Availability status changed
if old_t.get('success') != new_t.get('success'):
return True
# Rate changed significantly
old_rate_val = float(old_t.get('rate') or 0)
new_rate_val = float(new_t.get('rate') or 0)
if abs(old_rate_val - new_rate_val) > 0.01:
return True
# Min stay changed
if old_t.get('min_stay') != new_t.get('min_stay'):
return True
# Multi-night availability changed
if old_t.get('available_for_min_stay') != new_t.get('available_for_min_stay'):
return True
return False
def save_rate_snapshot(db, category_id: str, rate_date: date, rate: Dict) -> str:
"""
Save rate to database using snapshot logic.
If rate has changed from latest version, insert new row.
If rate is the same, just update last_verified_at.
Returns: 'inserted', 'verified', or 'error'
"""
gross_rate = rate.get('gross_rate')
net_rate = rate.get('net_rate')
tariffs_data = rate.get('tariffs_data', {})
# Get the latest rate for this category/date
existing = db.execute(
text("""
SELECT id, rate_gross, rate_net, tariffs_data
FROM newbook_current_rates
WHERE category_id = :category_id AND rate_date = :rate_date
ORDER BY valid_from DESC
LIMIT 1
"""),
{"category_id": category_id, "rate_date": rate_date}
).fetchone()
if existing:
existing_dict = {
'rate_gross': existing.rate_gross,
'rate_net': existing.rate_net,
'tariffs_data': existing.tariffs_data
}
else:
existing_dict = None
if rates_changed(existing_dict, rate):
# Rates changed - insert new snapshot
db.execute(
text("""
INSERT INTO newbook_current_rates
(category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from, last_verified_at)
VALUES (:category_id, :rate_date, :rate_gross, :rate_net,
CAST(:tariffs_data AS jsonb), NOW(), NOW())
"""),
{
"category_id": category_id,
"rate_date": rate_date,
"rate_gross": gross_rate,
"rate_net": net_rate,
"tariffs_data": json.dumps(tariffs_data)
}
)
return 'inserted'
else:
# Rates unchanged - just verify
db.execute(
text("""
UPDATE newbook_current_rates
SET last_verified_at = NOW()
WHERE id = :id
"""),
{"id": existing.id}
)
return 'verified'
def needs_multi_night_check(tariff: Dict, days_ahead: int) -> Optional[int]:
"""
Check if a tariff needs multi-night verification.
Returns the min_stay value if a multi-night check is needed, None otherwise.
Skips tariffs with advance booking restrictions that aren't met.
"""
min_stay = tariff.get('min_stay')
if not min_stay or min_stay <= 1:
return None
if tariff.get('success', False):
return None # Already available as single-night, no recheck needed
# Check for advance booking requirement
message = tariff.get('message', '') or ''
advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
if advance_match:
min_advance_days = int(advance_match.group(1))
if days_ahead < min_advance_days:
return None # Within advance period - recheck won't help
return min_stay
async def run_fetch_current_rates(horizon_days: int = 720, start_date: date = None):
"""
Fetch current rates for all included categories and store in database.
Args:
horizon_days: Number of days ahead to fetch (default 720 for scheduled, configurable for manual)
start_date: Start date for fetch (default today)
Processing: Day-by-day with progressive commits.
For each date:
1. Fetch single-night rates (all categories in one API call)
2. Check if any tariffs need multi-night verification
3. If so, run multi-night check immediately for that date
4. Save all rates for that date to DB
5. Commit every COMMIT_BATCH_SIZE days
This means if the job fails at day 400, the first 390+ days are already saved.
"""
logger.info(f"Starting current rates fetch ({horizon_days} days)")
db = next(iter([SyncSessionLocal()]))
today = start_date or date.today()
try:
# Get VAT rate from config
vat_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
).fetchone()
vat_rate_str = vat_result.config_value if vat_result and vat_result.config_value else '0.20'
# Get all included room categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = set(row.site_id for row in cat_result.fetchall())
if not included_categories:
logger.warning("No included room categories found")
return
logger.info(f"Fetching rates for {len(included_categories)} categories")
# Import rates client
import base64
from services.newbook_rates_client import NewbookRatesClient
# Credentials: central Settings service first, app-local config fallback
from services.central_settings import get_newbook_credentials_sync
creds = get_newbook_credentials_sync()
if not creds:
config_result = db.execute(
text("""
SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted
FROM system_config
WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region')
""")
)
config = {}
for row in config_result.fetchall():
value = row.config_value
if row.is_encrypted and value:
try:
value = base64.b64decode(value.encode()).decode()
except Exception:
pass # Use raw value if decryption fails
config[row.config_key] = value
if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
logger.error("Newbook credentials not configured")
return
creds = {
'api_key': config['newbook_api_key'],
'username': config['newbook_username'],
'password': config['newbook_password'],
'region': config['newbook_region'],
}
# Create client
client = NewbookRatesClient(
api_key=creds['api_key'],
username=creds['username'],
password=creds['password'],
region=creds['region'],
vat_rate=Decimal(vat_rate_str)
)
async with client:
inserted_total = 0
verified_total = 0
multi_night_checks = 0
skipped_advance = 0
current_date = today
day_count = 0
while current_date <= today + timedelta(days=horizon_days):
day_count += 1
days_ahead = (current_date - today).days
try:
# Step 1: Fetch single-night rates for all categories on this date
day_rates = await client.fetch_single_date_all_categories(
current_date, guests_adults=2, guests_children=0
)
# Step 2: Check for multi-night verification needs and run inline
# Collect unique min_stay values needed for this date
nights_needed: Set[int] = set()
for cat_id, rates in day_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
check = needs_multi_night_check(tariff, days_ahead)
if check:
nights_needed.add(check)
elif tariff.get('min_stay') and tariff['min_stay'] > 1 and not tariff.get('success', False):
skipped_advance += 1
# Step 3: Run multi-night checks for this date if needed
multi_night_results: Dict[int, Dict[str, Dict[str, bool]]] = {}
for nights in sorted(nights_needed):
try:
result = await client.fetch_multi_night_for_date(
current_date, nights
)
multi_night_results[nights] = result
multi_night_checks += 1
await asyncio.sleep(1.0) # Rate limiting
except Exception as e:
logger.warning(f"Multi-night check failed for {current_date} ({nights}n): {e}")
# Step 4: Update tariffs with multi-night results and save to DB
for cat_id, rates in day_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
tariffs_data = rate.get('tariffs_data', {})
# Apply multi-night results to tariffs
for tariff in tariffs_data.get('tariffs', []):
min_stay = tariff.get('min_stay')
if min_stay and min_stay > 1 and min_stay in multi_night_results:
cat_availability = multi_night_results[min_stay].get(cat_id, {})
tariff_name = tariff.get('name', '')
tariff['available_for_min_stay'] = cat_availability.get(tariff_name, False)
# Save to DB
result = save_rate_snapshot(db, cat_id, current_date, rate)
if result == 'inserted':
inserted_total += 1
elif result == 'verified':
verified_total += 1
if day_count % 50 == 0 or nights_needed:
logger.info(
f"Day {day_count}/{horizon_days}: {current_date}"
f" | {inserted_total} new, {verified_total} verified"
f"{f' | {len(nights_needed)} multi-night checks' if nights_needed else ''}"
)
except Exception as e:
logger.warning(f"Failed to fetch rates for {current_date}: {e}")
# Step 5: Commit periodically
if day_count % COMMIT_BATCH_SIZE == 0:
db.commit()
current_date += timedelta(days=1)
await asyncio.sleep(1.0) # Rate limiting between days
# Final commit for remaining days
db.commit()
if skipped_advance > 0:
logger.info(f"Skipped {skipped_advance} multi-night checks (advance booking restriction)")
logger.info(
f"Complete: {inserted_total} new snapshots, {verified_total} verified unchanged, "
f"{multi_night_checks} multi-night checks"
)
logger.info("Current rates fetch completed successfully")
except Exception as e:
logger.error(f"Current rates fetch failed: {e}")
db.rollback()
raise
finally:
db.close()

165
backend/jobs/scrape_booking_rates.py Normal file
View file

@ -0,0 +1,165 @@
"""
Scheduled Booking.com Rate Scraping Job
Priority-based scheduling for 365-day coverage (all queued daily):
- High (priority 10): next 30 days
- Medium (priority 5): days 31-180
- Low (priority 2): days 181-365
Queue processes in priority order. If rate-limited/blocked, lower priority
dates remain queued for the next run.
Uses a queue-based approach:
1. Populate the queue with dates and priorities
2. Process the queue in priority order
3. Failed dates are retried (up to 3 attempts)
4. On blocking, the queue pauses and resumes after cooldown
Schedule: Daily at configurable time (default 05:30)
"""
import asyncio
import logging
from datetime import date, timedelta
from sqlalchemy import text
from database import SyncSessionLocal
from services.booking_scraper import (
populate_queue,
process_queue,
clear_old_queue_items,
cleanup_stale_batches,
get_scrape_config,
)
logger = logging.getLogger(__name__)
# Priority levels (higher = processed first)
PRIORITY_HIGH = 10 # 0-30 days
PRIORITY_MEDIUM = 5 # 31-180 days
PRIORITY_LOW = 2 # 181-365 days
def get_high_priority_dates() -> list[date]:
"""High priority: today + 30 days."""
today = date.today()
return [today + timedelta(days=i) for i in range(31)]
def get_medium_priority_dates() -> list[date]:
"""Medium priority: days 31-180."""
today = date.today()
return [today + timedelta(days=i) for i in range(31, 181)]
def get_low_priority_dates() -> list[date]:
"""Low priority: days 181-365."""
today = date.today()
return [today + timedelta(days=i) for i in range(181, 366)]
def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
"""
For a target date, determine its priority tier and when it will next be scraped.
Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'.
All dates are queued daily, so next scrape is always today (or tomorrow if
today's run has passed).
"""
today = date.today()
offset = (target_date - today).days
if offset < 0:
return ('none', None)
if offset > 365:
return ('none', None)
# All tiers run daily - next scrape is today
if offset <= 30:
return ('high', today)
elif offset <= 180:
return ('medium', today)
else:
return ('low', today)
def run_scheduled_booking_scrape():
"""
Main scheduled job: populate queue with today's dates, then process.
"""
db = SyncSessionLocal()
try:
# Check if scraper is enabled
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
).fetchone()
if not result or result.config_value != 'true':
logger.debug("Scheduled booking scrape skipped (disabled)")
return
if not get_scrape_config(db):
logger.warning("Scheduled booking scrape skipped (no location configured)")
return
# Clean up stale running batches and old queue items
cleanup_stale_batches(db, max_age_minutes=120)
clear_old_queue_items(db, days=3)
# Gather dates with priorities
high = get_high_priority_dates()
medium = get_medium_priority_dates()
low = get_low_priority_dates()
priorities = {}
for d in high:
priorities[d] = PRIORITY_HIGH
for d in medium:
priorities[d] = max(priorities.get(d, 0), PRIORITY_MEDIUM)
for d in low:
priorities[d] = max(priorities.get(d, 0), PRIORITY_LOW)
all_dates = sorted(priorities.keys())
if not all_dates:
logger.info("Scheduled booking scrape: no dates to scrape today")
return
logger.info(
f"Scheduled booking scrape: queuing {len(all_dates)} dates "
f"(high={len(high)}, medium={len(medium)}, low={len(low)})"
)
# Populate queue
populate_queue(db, all_dates, priorities)
# Process queue
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
result = loop.run_until_complete(process_queue(db))
if result.get('success'):
logger.info(
f"Scheduled booking scrape completed: "
f"{result.get('dates_completed', 0)} dates, "
f"{result.get('rates_scraped', 0)} rates"
)
elif result.get('blocked'):
logger.warning(
f"Scheduled booking scrape blocked: {result.get('block_reason')}. "
f"Completed {result.get('dates_completed', 0)} dates. "
f"Remaining dates stay queued for retry."
)
else:
logger.error(f"Scheduled booking scrape failed: {result.get('error')}")
finally:
loop.close()
except Exception as e:
logger.error(f"Scheduled booking scrape error: {e}", exc_info=True)
finally:
db.close()
async def run_scheduled_booking_scrape_async():
"""Async wrapper for APScheduler."""
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, run_scheduled_booking_scrape)

43
backend/jobs/scrape_direct_rates.py Normal file
View file

@ -0,0 +1,43 @@
"""
Daily job: scrape all enabled direct competitor hotels.
Hotels are scraped sequentially — the scraper enforces 10s delays per date
to avoid rate-limiting, so concurrent scraping is not beneficial.
"""
import logging
from sqlalchemy import text
from database import SyncSessionLocal
from services.direct_scraper import run_scrape
log = logging.getLogger(__name__)
def run_scrape_all_direct():
db = SyncSessionLocal()
try:
rows = db.execute(
text("""SELECT id, name, profile_name, params
FROM direct_competitor_hotels
WHERE scrape_enabled = true
ORDER BY id""")
).mappings().fetchall()
finally:
db.close()
if not rows:
log.info("No direct competitor hotels enabled for scraping")
return
log.info(f"Starting direct rate scrape for {len(rows)} hotels")
for hotel in rows:
try:
log.info(f"Scraping {hotel['name']} ({hotel['profile_name']})")
run_scrape(
hotel_id=hotel["id"],
profile_name=hotel["profile_name"],
params=hotel["params"],
)
except Exception as e:
log.error(f"Direct scrape failed for hotel {hotel['id']} ({hotel['name']}): {e}")
log.info("Direct rate scrape complete")

80
backend/main.py Normal file
View file

@ -0,0 +1,80 @@
"""
Rate Monitor — FastAPI Backend
"""
import logging
import sys
from contextlib import asynccontextmanager
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[logging.StreamHandler(sys.stdout)]
)
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from database import DATABASE_URL
from api import bookability, competitors, analysis, direct
from scheduler import start_scheduler, shutdown_scheduler
@asynccontextmanager
async def lifespan(app: FastAPI):
try:
import os, psycopg2
schema_path = os.path.join(os.path.dirname(__file__), 'schema.sql')
if os.path.exists(schema_path):
with open(schema_path) as f:
sql = f.read()
conn = psycopg2.connect(DATABASE_URL)
try:
conn.autocommit = True
with conn.cursor() as cur:
cur.execute(sql)
logging.getLogger(__name__).info("Schema applied successfully")
finally:
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Schema init failed: {e}")
try:
from services.booking_scraper import cleanup_stale_batches
from database import SyncSessionLocal
db = SyncSessionLocal()
try:
cleanup_stale_batches(db, max_age_minutes=10)
finally:
db.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Stale batch cleanup failed: {e}")
start_scheduler()
yield
shutdown_scheduler()
app = FastAPI(
title="Rate Monitor API",
description="Rate monitoring, competitor scraping, and bookability service",
version="1.0.0",
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(bookability.router, prefix="/bookability", tags=["Bookability"])
app.include_router(competitors.router, prefix="/competitors", tags=["Competitors"])
app.include_router(analysis.router, prefix="/analysis", tags=["Rate Analysis"])
app.include_router(direct.router, prefix="/direct", tags=["Direct Rates"])
@app.get("/health")
async def health_check():
return {"status": "healthy", "service": "rate-monitor-api"}

167
backend/migrate_direct_data.py Normal file
View file

@ -0,0 +1,167 @@
"""
One-off migration: copy guestline-monitor SQLite data into PostgreSQL.
Run manually after deploy:
docker compose exec backend python migrate_direct_data.py
"""
import json
import os
import sqlite3
import sys
from datetime import datetime, timezone
import psycopg2
ARCHIVE_DIR = os.environ.get(
"GUESTLINE_ARCHIVE",
"/home/jtr/laptop-archive/guestline-monitor/data"
)
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://rates:rates_secret@localhost:5432/rates_db")
CONFIG_DB = os.path.join(ARCHIVE_DIR, "config.db")
def dict_row(cursor, row):
return {col[0]: val for col, val in zip(cursor.description, row)}
def migrate():
if not os.path.exists(CONFIG_DB):
print(f"ERROR: config.db not found at {CONFIG_DB}", file=sys.stderr)
print("Set GUESTLINE_ARCHIVE env var to the data directory path.")
sys.exit(1)
pg = psycopg2.connect(DATABASE_URL)
pg.autocommit = False
# ── Read hotel configs from SQLite ──────────────────────────────────────
src = sqlite3.connect(CONFIG_DB)
src.row_factory = dict_row
hotels = src.execute(
"""SELECT id, name, profile, params, room_labels, rate_labels,
room_order, benchmark_room, benchmark_rate,
tier_base_room, tier_offsets, scrape_enabled, last_scraped_at
FROM hotels ORDER BY id"""
).fetchall()
src.close()
if not hotels:
print("No hotels found in config.db — nothing to migrate.")
return
print(f"Migrating {len(hotels)} hotels...")
# Map old SQLite hotel IDs to new PostgreSQL IDs
id_map: dict[int, int] = {}
with pg.cursor() as cur:
for h in hotels:
params = h["params"] if isinstance(h["params"], str) else json.dumps(h["params"] or {})
cur.execute(
"""INSERT INTO direct_competitor_hotels
(name, profile_name, params, room_labels, rate_labels, room_order,
benchmark_room, benchmark_rate, tier_base_room, tier_offsets,
scrape_enabled, last_scraped_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id""",
(
h["name"],
h["profile"],
params,
h["room_labels"] or "{}",
h["rate_labels"] or "{}",
h["room_order"] or "[]",
h["benchmark_room"],
h["benchmark_rate"],
h["tier_base_room"],
h["tier_offsets"] or "{}",
bool(h["scrape_enabled"]),
h["last_scraped_at"],
)
)
new_id = cur.fetchone()[0]
id_map[h["id"]] = new_id
print(f" Hotel {h['id']} → {new_id}: {h['name']}")
pg.commit()
# ── Migrate snapshot data from per-hotel SQLite DBs ─────────────────────
total_rows = 0
for old_id, new_id in id_map.items():
db_path = os.path.join(ARCHIVE_DIR, f"hotel_{old_id}.db")
if not os.path.exists(db_path):
print(f" hotel_{old_id}.db not found — skipping snapshot data")
continue
src = sqlite3.connect(db_path)
src.row_factory = dict_row
# Migrate scrape_runs first
runs = src.execute(
"SELECT id, scraped_at, dates_found, rows_saved FROM scrape_runs ORDER BY id"
).fetchall()
run_id_map: dict[int, int] = {}
with pg.cursor() as cur:
for run in runs:
cur.execute(
"""INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found, rows_saved)
VALUES (%s, %s, %s, %s) RETURNING id""",
(new_id, run["scraped_at"], run["dates_found"] or 0, run["rows_saved"] or 0)
)
run_id_map[run["id"]] = cur.fetchone()[0]
pg.commit()
# Migrate snapshots in batches
BATCH = 2000
offset = 0
hotel_rows = 0
while True:
rows = src.execute(
"""SELECT scrape_run_id, scraped_at, stay_date, room_id, rate_id,
availability, price_excl, price_incl, currency, min_stay_nights
FROM snapshots ORDER BY id LIMIT ? OFFSET ?""",
(BATCH, offset)
).fetchall()
if not rows:
break
with pg.cursor() as cur:
psycopg2.extras.execute_values(
cur,
"""INSERT INTO direct_rates
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
availability, price_excl, price_incl, currency, min_stay_nights)
VALUES %s""",
[
(
new_id,
run_id_map.get(r["scrape_run_id"]),
r["scraped_at"],
r["stay_date"],
r["room_id"],
r["rate_id"],
r["availability"] or 0,
r["price_excl"],
r["price_incl"],
r["currency"] or "GBP",
r["min_stay_nights"],
)
for r in rows
]
)
pg.commit()
hotel_rows += len(rows)
offset += BATCH
print(f" {hotel_rows} rows migrated for hotel {old_id}...", end="\r")
print(f" hotel_{old_id}: {hotel_rows} snapshot rows migrated")
total_rows += hotel_rows
src.close()
pg.close()
print(f"\nMigration complete: {len(hotels)} hotels, {total_rows} snapshot rows.")
if __name__ == "__main__":
import psycopg2.extras
migrate()

14
backend/requirements.txt Normal file
View file

@ -0,0 +1,14 @@
fastapi==0.109.0
uvicorn[standard]==0.27.0
sqlalchemy==2.0.25
asyncpg==0.29.0
psycopg2-binary==2.9.9
python-jose[cryptography]==3.3.0
python-multipart==0.0.6
httpx==0.26.0
apscheduler==3.10.4
pydantic==2.5.3
pydantic-settings==2.1.0
python-dotenv==1.0.0
python-dateutil==2.8.2
playwright>=1.40.0

120
backend/scheduler.py Normal file
View file

@ -0,0 +1,120 @@
"""
APScheduler configuration for rate monitor jobs
"""
import logging
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
scheduler = AsyncIOScheduler(
job_defaults={
'misfire_grace_time': 3600,
'coalesce': True,
}
)
def get_config_value(key: str, default: str = None) -> str:
db = SyncSessionLocal()
try:
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = :key"),
{"key": key}
)
row = result.fetchone()
if row and row.config_value:
return row.config_value
return default
except Exception as e:
logger.error(f"Error getting config {key}: {e}")
return default
finally:
db.close()
def is_sync_enabled(source: str) -> bool:
value = get_config_value(f"sync_{source}_enabled")
if value:
return value.lower() in ('true', '1', 'yes', 'enabled')
return False
def get_sync_time(source: str, default_hour: int = 5, default_minute: int = 0) -> tuple:
time_str = get_config_value(f"sync_{source}_time")
if time_str:
try:
parts = time_str.split(':')
return (int(parts[0]), int(parts[1]))
except (ValueError, IndexError):
pass
return (default_hour, default_minute)
async def run_scheduled_direct_scrape():
from jobs.scrape_direct_rates import run_scrape_all_direct
import asyncio
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, run_scrape_all_direct)
async def run_scheduled_booking_scrape_async():
from jobs.scrape_booking_rates import run_scheduled_booking_scrape
import asyncio
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, run_scheduled_booking_scrape)
async def run_scheduled_fetch_current_rates():
if is_sync_enabled("newbook_current_rates"):
from jobs.fetch_current_rates import run_fetch_current_rates
import asyncio
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, run_fetch_current_rates)
else:
logger.debug("Newbook current rates sync skipped (disabled)")
def start_scheduler():
# Booking.com scrape — daily at configurable time (default 05:30)
scrape_time = get_config_value('booking_scraper_daily_time', '05:30')
try:
h, m = scrape_time.split(':')
scrape_hour, scrape_minute = int(h), int(m)
except Exception:
scrape_hour, scrape_minute = 5, 30
scheduler.add_job(
run_scheduled_booking_scrape_async,
CronTrigger(hour=scrape_hour, minute=scrape_minute),
id='booking_scrape',
replace_existing=True,
)
# Newbook current rates fetch — daily at 05:20
rates_hour, rates_minute = get_sync_time('newbook_current_rates', 5, 20)
scheduler.add_job(
run_scheduled_fetch_current_rates,
CronTrigger(hour=rates_hour, minute=rates_minute),
id='fetch_current_rates',
replace_existing=True,
)
# Direct booking engine scrape — daily at 06:00
scheduler.add_job(
run_scheduled_direct_scrape,
CronTrigger(hour=6, minute=0),
id='scrape_direct_rates',
replace_existing=True,
)
scheduler.start()
logger.info(f"Scheduler started: booking scrape at {scrape_hour:02d}:{scrape_minute:02d}, rates fetch at {rates_hour:02d}:{rates_minute:02d}, direct scrape at 06:00")
def shutdown_scheduler():
if scheduler.running:
scheduler.shutdown()

267
backend/schema.sql Normal file
View file

@ -0,0 +1,267 @@
-- Rate Monitor — Database Schema
-- All CREATE TABLE/INDEX are idempotent (IF NOT EXISTS)
-- ============================================
-- SYSTEM CONFIG
-- ============================================
CREATE TABLE IF NOT EXISTS system_config (
config_key VARCHAR(100) PRIMARY KEY,
config_value TEXT,
description TEXT,
updated_at TIMESTAMPTZ DEFAULT NOW(),
updated_by VARCHAR(255)
);
INSERT INTO system_config (config_key, config_value, description) VALUES
('booking_scraper_enabled', 'false', 'Enable automatic booking.com rate scraping (true/false)'),
('booking_scraper_paused', 'false', 'Scraper temporarily paused due to blocking (true/false)'),
('booking_scraper_pause_until', NULL, 'ISO datetime when pause expires'),
('booking_scraper_backend', 'playwright_local', 'Scraper backend: playwright_local | playwright_proxy | apify'),
('booking_scraper_daily_time', '05:30', 'Daily scrape time (HH:MM)'),
('booking_scraper_proxy_url', NULL, 'Proxy URL for playwright_proxy backend'),
('booking_scraper_proxy_username', NULL, 'Proxy username for playwright_proxy backend'),
('booking_scraper_proxy_password', NULL, 'Proxy password for playwright_proxy backend'),
('sync_newbook_current_rates_enabled', 'false', 'Enable automatic Newbook current rates sync (true/false)'),
('sync_newbook_current_rates_time', '05:20', 'Newbook current rates sync time (HH:MM)')
ON CONFLICT (config_key) DO NOTHING;
-- ============================================
-- BOOKING.COM SCRAPE LOG (batch tracking)
-- ============================================
CREATE TABLE IF NOT EXISTS booking_scrape_log (
batch_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
scrape_type VARCHAR(50) NOT NULL,
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
status VARCHAR(20) NOT NULL DEFAULT 'running',
hotels_found INTEGER DEFAULT 0,
rates_scraped INTEGER DEFAULT 0,
dates_queued INTEGER DEFAULT 0,
dates_completed INTEGER DEFAULT 0,
dates_failed INTEGER DEFAULT 0,
error_message TEXT,
blocked_at TIMESTAMPTZ,
resume_after TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_booking_scrape_log_started ON booking_scrape_log(started_at DESC);
-- ============================================
-- BOOKING.COM SCRAPE CONFIG
-- ============================================
CREATE TABLE IF NOT EXISTS booking_scrape_config (
id SERIAL PRIMARY KEY,
location_name VARCHAR(255) NOT NULL,
location_search_url TEXT,
pages_to_scrape INTEGER DEFAULT 2,
adults INTEGER DEFAULT 2,
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- ============================================
-- BOOKING.COM SCRAPE QUEUE
-- ============================================
CREATE TABLE IF NOT EXISTS booking_scrape_queue (
id SERIAL PRIMARY KEY,
rate_date DATE NOT NULL,
status VARCHAR(20) DEFAULT 'pending',
priority INTEGER DEFAULT 0,
attempts INTEGER DEFAULT 0,
max_attempts INTEGER DEFAULT 3,
last_attempt_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT NOW(),
error_message TEXT,
UNIQUE (rate_date, status)
);
CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending
ON booking_scrape_queue(status, priority DESC, rate_date ASC)
WHERE status = 'pending';
-- ============================================
-- BOOKING.COM HOTELS
-- ============================================
CREATE TABLE IF NOT EXISTS booking_com_hotels (
id SERIAL PRIMARY KEY,
booking_com_id VARCHAR(50) UNIQUE,
name VARCHAR(255) NOT NULL,
booking_com_url TEXT,
star_rating DECIMAL(2,1),
review_score DECIMAL(3,1),
review_count INTEGER DEFAULT 0,
tier VARCHAR(20) DEFAULT 'market', -- own | competitor | market
display_order INTEGER DEFAULT 999,
notes TEXT,
is_active BOOLEAN DEFAULT TRUE,
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
last_seen_at TIMESTAMPTZ DEFAULT NOW()
);
-- ============================================
-- BOOKING.COM RATES
-- ============================================
CREATE TABLE IF NOT EXISTS booking_com_rates (
id SERIAL PRIMARY KEY,
hotel_id INTEGER NOT NULL REFERENCES booking_com_hotels(id) ON DELETE CASCADE,
rate_date DATE NOT NULL,
availability_status VARCHAR(30), -- available | sold_out | no_data
rate_gross DECIMAL(10,2),
currency VARCHAR(10) DEFAULT 'GBP',
room_type VARCHAR(100),
breakfast_included BOOLEAN DEFAULT FALSE,
free_cancellation BOOLEAN DEFAULT FALSE,
no_prepayment BOOLEAN DEFAULT FALSE,
rooms_left INTEGER,
scrape_batch_id UUID REFERENCES booking_scrape_log(batch_id),
scraped_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);
-- ============================================
-- RATE PARITY ALERTS
-- ============================================
CREATE TABLE IF NOT EXISTS rate_parity_alerts (
id SERIAL PRIMARY KEY,
rate_date DATE NOT NULL,
room_category VARCHAR(255),
newbook_rate DECIMAL(12,2),
booking_com_rate DECIMAL(12,2),
difference_pct DECIMAL(8,2),
alert_type VARCHAR(20), -- higher | lower
alert_status VARCHAR(20) DEFAULT 'active', -- active | acknowledged
created_at TIMESTAMPTZ DEFAULT NOW(),
acknowledged_at TIMESTAMPTZ,
acknowledged_by VARCHAR(255),
notes TEXT
);
CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_date ON rate_parity_alerts(rate_date DESC);
CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_status ON rate_parity_alerts(alert_status);
-- ============================================
-- NEWBOOK CURRENT RATES (own hotel, for bookability display)
-- ============================================
CREATE TABLE IF NOT EXISTS newbook_current_rates (
id SERIAL PRIMARY KEY,
category_id VARCHAR(50) NOT NULL,
rate_date DATE NOT NULL,
gross_rate DECIMAL(12,2),
net_rate DECIMAL(12,2),
tariffs_data JSONB DEFAULT '{}',
valid_from TIMESTAMP DEFAULT NOW(),
last_verified_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_current_rates_date ON newbook_current_rates(rate_date);
CREATE INDEX IF NOT EXISTS idx_current_rates_category ON newbook_current_rates(category_id);
CREATE INDEX IF NOT EXISTS idx_current_rates_tariffs ON newbook_current_rates USING gin(tariffs_data);
CREATE INDEX IF NOT EXISTS idx_current_rates_latest ON newbook_current_rates(category_id, rate_date, valid_from DESC);
-- ============================================
-- NEWBOOK ROOM CATEGORIES (for bookability display)
-- ============================================
CREATE TABLE IF NOT EXISTS newbook_room_categories (
id SERIAL PRIMARY KEY,
site_id VARCHAR(50) NOT NULL UNIQUE,
site_name VARCHAR(255) NOT NULL,
site_type VARCHAR(100),
room_count INTEGER DEFAULT 0,
is_included BOOLEAN DEFAULT TRUE,
display_order INTEGER DEFAULT 0,
fetched_at TIMESTAMP DEFAULT NOW()
);
-- ============================================
-- NEWBOOK OCCUPANCY REPORT DATA
-- ============================================
CREATE TABLE IF NOT EXISTS newbook_occupancy_report_data (
id SERIAL PRIMARY KEY,
date DATE NOT NULL,
category_id VARCHAR(50) NOT NULL,
category_name VARCHAR(255),
available INTEGER DEFAULT 0,
occupied INTEGER DEFAULT 0,
maintenance INTEGER DEFAULT 0,
allotted INTEGER DEFAULT 0,
revenue_gross DECIMAL(12,2) DEFAULT 0,
revenue_net DECIMAL(12,2) DEFAULT 0,
occupancy_pct DECIMAL(5,2),
fetched_at TIMESTAMP DEFAULT NOW(),
UNIQUE(date, category_id)
);
CREATE INDEX IF NOT EXISTS idx_occupancy_report_date ON newbook_occupancy_report_data(date);
-- ============================================
-- DIRECT COMPETITOR HOTEL CONFIGS
-- ============================================
CREATE TABLE IF NOT EXISTS direct_competitor_hotels (
id SERIAL PRIMARY KEY,
name VARCHAR(255) NOT NULL,
profile_name VARCHAR(50) NOT NULL,
params JSONB NOT NULL DEFAULT '{}',
room_labels JSONB DEFAULT '{}',
rate_labels JSONB DEFAULT '{}',
room_order JSONB DEFAULT '[]',
benchmark_room VARCHAR(100),
benchmark_rate VARCHAR(100),
tier_base_room VARCHAR(100),
tier_offsets JSONB DEFAULT '{}',
scrape_enabled BOOLEAN DEFAULT TRUE,
last_scraped_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- ============================================
-- DIRECT SCRAPE RUNS
-- ============================================
CREATE TABLE IF NOT EXISTS direct_scrape_runs (
id SERIAL PRIMARY KEY,
hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE,
scraped_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
dates_found INTEGER DEFAULT 0,
rows_saved INTEGER DEFAULT 0
);
-- ============================================
-- DIRECT RATES (per-room snapshots from booking engines)
-- ============================================
CREATE TABLE IF NOT EXISTS direct_rates (
id SERIAL PRIMARY KEY,
hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE,
scrape_run_id INTEGER REFERENCES direct_scrape_runs(id),
scraped_at TIMESTAMPTZ NOT NULL,
stay_date DATE NOT NULL,
room_id VARCHAR(100) NOT NULL,
rate_id VARCHAR(100) NOT NULL,
availability INTEGER DEFAULT 0,
price_excl DECIMAL(10,2),
price_incl DECIMAL(10,2),
currency VARCHAR(10) DEFAULT 'GBP',
min_stay_nights INTEGER
);
CREATE INDEX IF NOT EXISTS idx_direct_rates_hotel_date ON direct_rates(hotel_id, stay_date);
CREATE INDEX IF NOT EXISTS idx_direct_rates_scraped ON direct_rates(scraped_at DESC);
-- Link booking_com_hotels to direct_competitor_hotels (optional, for Market View direct column)
ALTER TABLE booking_com_hotels ADD COLUMN IF NOT EXISTS direct_hotel_id INTEGER
REFERENCES direct_competitor_hotels(id) ON DELETE SET NULL;

0
backend/services/__init__.py Normal file
View file

829
backend/services/booking_scraper.py Normal file
View file

@ -0,0 +1,829 @@
"""
Booking.com Rate Scraper Service
Main service for scraping competitor rates from booking.com.
Uses pluggable backends (Playwright local, proxy, Apify) via factory pattern.
Features:
- Location-based search (1 query = 40+ hotels)
- Hotel discovery and tier management
- Rate extraction with availability status
- Anti-scrape detection and pause/resume
"""
import logging
import uuid
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import List, Optional, Dict, Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
logger = logging.getLogger(__name__)
def get_scraper_backend(db: Session) -> ScraperBackend:
"""
Factory to get configured scraper backend.
Reads backend type from system_config and returns appropriate instance.
"""
# Get backend configuration
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'")
).fetchone()
backend_type = result.config_value if result and result.config_value else 'playwright_local'
if backend_type == 'playwright_local':
return PlaywrightLocalBackend()
elif backend_type == 'playwright_proxy':
# Get proxy config
proxy_result = db.execute(
text("""
SELECT config_key, config_value FROM system_config
WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password')
""")
)
proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()}
return PlaywrightLocalBackend(proxy_config=proxy_config)
elif backend_type == 'apify':
# Future: Apify backend
raise NotImplementedError("Apify backend not yet implemented")
else:
logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
return PlaywrightLocalBackend()
def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
"""Get the active scrape location configuration."""
result = db.execute(
text("""
SELECT id, location_name, location_search_url, pages_to_scrape, adults
FROM booking_scrape_config
WHERE is_active = TRUE
ORDER BY id
LIMIT 1
""")
).fetchone()
if not result:
return None
return {
'id': result.id,
'location_name': result.location_name,
'location_search_url': result.location_search_url,
'pages_to_scrape': result.pages_to_scrape or 2,
'adults': result.adults or 2,
}
async def is_scraper_paused(db: Session) -> bool:
"""Check if scraper is currently paused due to blocking."""
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
).fetchone()
if not result or result.config_value != 'true':
return False
# Check if pause period has expired
pause_until_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'")
).fetchone()
if pause_until_result and pause_until_result.config_value:
try:
pause_until = datetime.fromisoformat(pause_until_result.config_value)
if datetime.now() >= pause_until:
# Pause expired, reset
db.execute(
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
)
db.commit()
return False
except ValueError:
pass
return True
async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
"""Set scraper pause status."""
db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
{'val': 'true' if paused else 'false'}
)
if paused:
pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
db.execute(
text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
{'val': pause_until}
)
db.commit()
def save_hotel(db: Session, hotel: HotelData) -> int:
"""
Save or update a hotel in the database.
Returns the hotel's database ID.
"""
# Check if hotel exists
existing = db.execute(
text("SELECT id FROM booking_com_hotels WHERE booking_com_id = :bid"),
{'bid': hotel.booking_com_id}
).fetchone()
if existing:
# Update last_seen_at and any changed fields
db.execute(
text("""
UPDATE booking_com_hotels SET
name = COALESCE(:name, name),
booking_com_url = COALESCE(:url, booking_com_url),
star_rating = COALESCE(:stars, star_rating),
review_score = COALESCE(:score, review_score),
review_count = COALESCE(:count, review_count),
last_seen_at = NOW()
WHERE booking_com_id = :bid
"""),
{
'bid': hotel.booking_com_id,
'name': hotel.name,
'url': hotel.booking_com_url,
'stars': float(hotel.star_rating) if hotel.star_rating else None,
'score': float(hotel.review_score) if hotel.review_score else None,
'count': hotel.review_count,
}
)
return existing.id
else:
# Insert new hotel (default tier is 'market')
result = db.execute(
text("""
INSERT INTO booking_com_hotels
(booking_com_id, name, booking_com_url, star_rating, review_score, review_count, tier)
VALUES (:bid, :name, :url, :stars, :score, :count, 'market')
RETURNING id
"""),
{
'bid': hotel.booking_com_id,
'name': hotel.name,
'url': hotel.booking_com_url,
'stars': float(hotel.star_rating) if hotel.star_rating else None,
'score': float(hotel.review_score) if hotel.review_score else None,
'count': hotel.review_count,
}
)
return result.fetchone().id
def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
"""Save a rate to the database."""
db.execute(
text("""
INSERT INTO booking_com_rates
(hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id)
VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
:breakfast, :cancel, :prepay, :rooms_left, :batch_id)
"""),
{
'hotel_id': hotel_id,
'rate_date': rate.rate_date,
'status': rate.availability_status.value,
'rate': float(rate.rate_gross) if rate.rate_gross else None,
'currency': rate.currency,
'room_type': rate.room_type,
'breakfast': rate.breakfast_included,
'cancel': rate.free_cancellation,
'prepay': rate.no_prepayment,
'rooms_left': rate.rooms_left,
'batch_id': str(batch_id),
}
)
def create_scrape_batch(db: Session, scrape_type: str) -> uuid.UUID:
"""Create a new scrape batch log entry."""
batch_id = uuid.uuid4()
db.execute(
text("""
INSERT INTO booking_scrape_log
(batch_id, scrape_type, started_at, status)
VALUES (:batch_id, :scrape_type, NOW(), 'running')
"""),
{'batch_id': str(batch_id), 'scrape_type': scrape_type}
)
db.commit()
return batch_id
def update_scrape_batch(
db: Session,
batch_id: uuid.UUID,
status: str,
hotels_found: int = 0,
rates_scraped: int = 0,
error_message: str = None,
blocked: bool = False
):
"""Update scrape batch log with results."""
db.execute(
text("""
UPDATE booking_scrape_log SET
completed_at = CASE WHEN :status IN ('completed', 'failed', 'blocked') THEN NOW() ELSE NULL END,
status = :status,
hotels_found = :hotels,
rates_scraped = :rates,
error_message = :error,
blocked_at = CASE WHEN :blocked THEN NOW() ELSE NULL END,
resume_after = CASE WHEN :blocked THEN NOW() + INTERVAL '2 hours' ELSE NULL END
WHERE batch_id = :batch_id
"""),
{
'batch_id': str(batch_id),
'status': status,
'hotels': hotels_found,
'rates': rates_scraped,
'error': error_message,
'blocked': blocked,
}
)
db.commit()
def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
"""
Mark any 'running' scrape batches as 'failed' if they've been running
longer than max_age_minutes. This handles orphaned batches from
container restarts or crashes.
"""
result = db.execute(
text("""
UPDATE booking_scrape_log SET
status = 'failed',
completed_at = NOW(),
error_message = 'Interrupted (container restart or timeout)'
WHERE status = 'running'
AND started_at < NOW() - INTERVAL ':mins minutes'
RETURNING batch_id
""".replace(':mins', str(int(max_age_minutes))))
)
cleaned = result.fetchall()
db.commit()
if cleaned:
logger.info(f"Cleaned up {len(cleaned)} stale running scrape batch(es)")
return len(cleaned)
async def scrape_date(
db: Session,
rate_date: date,
backend: ScraperBackend,
config: Dict[str, Any],
batch_id: uuid.UUID
) -> Dict[str, Any]:
"""
Scrape rates for a single date.
Args:
db: Database session
rate_date: Date to scrape rates for
backend: Scraper backend instance
config: Scrape configuration
batch_id: Current batch ID
Returns:
Dict with 'success', 'blocked', 'hotels_count', 'rates_count'
"""
check_in = rate_date
check_out = rate_date + timedelta(days=1) # Single night
result = await backend.scrape_location_search(
location=config['location_name'],
check_in=check_in,
check_out=check_out,
adults=config['adults'],
pages=config['pages_to_scrape']
)
if result.blocked:
return {
'success': False,
'blocked': True,
'block_reason': result.block_reason,
'hotels_count': 0,
'rates_count': 0,
}
if not result.success:
return {
'success': False,
'blocked': False,
'error': result.error_message,
'hotels_count': 0,
'rates_count': 0,
}
# Save hotels and rates
hotels_saved = 0
rates_saved = 0
for hotel, rate in zip(result.hotels, result.rates):
if not hotel.booking_com_id:
continue
try:
hotel_id = save_hotel(db, hotel)
save_rate(db, rate, hotel_id, batch_id)
hotels_saved += 1
rates_saved += 1
except Exception as e:
logger.warning(f"Error saving hotel/rate: {e}")
continue
db.commit()
return {
'success': True,
'blocked': False,
'hotels_count': hotels_saved,
'rates_count': rates_saved,
}
async def run_manual_scrape(
db: Session,
from_date: date,
to_date: date = None
) -> Dict[str, Any]:
"""
Run a manual scrape for testing/on-demand use.
Args:
db: Database session
from_date: Start date
to_date: End date (defaults to from_date for single day)
Returns:
Dict with scrape results summary
"""
if to_date is None:
to_date = from_date
# Check if paused
if await is_scraper_paused(db):
return {
'success': False,
'error': 'Scraper is currently paused due to blocking. Try again later.',
}
# Get config
config = get_scrape_config(db)
if not config:
return {
'success': False,
'error': 'No scrape location configured. Add a location in settings.',
}
# Create batch
batch_id = create_scrape_batch(db, 'manual')
# Get backend
backend = get_scraper_backend(db)
total_hotels = 0
total_rates = 0
dates_completed = 0
dates_failed = 0
try:
current_date = from_date
while current_date <= to_date:
logger.info(f"Scraping date: {current_date}")
result = await scrape_date(db, current_date, backend, config, batch_id)
if result['blocked']:
# Blocking detected - pause and exit
await set_scraper_paused(db, True, hours=2)
update_scrape_batch(
db, batch_id,
status='blocked',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
blocked=True
)
return {
'success': False,
'blocked': True,
'block_reason': result.get('block_reason'),
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
if result['success']:
total_hotels += result['hotels_count']
total_rates += result['rates_count']
dates_completed += 1
else:
dates_failed += 1
logger.warning(f"Failed to scrape {current_date}: {result.get('error')}")
current_date += timedelta(days=1)
# Update batch as completed
update_scrape_batch(
db, batch_id,
status='completed',
hotels_found=total_hotels,
rates_scraped=total_rates
)
return {
'success': True,
'blocked': False,
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
except Exception as e:
logger.error(f"Scrape error: {e}")
update_scrape_batch(
db, batch_id,
status='failed',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=str(e)
)
return {
'success': False,
'error': str(e),
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
finally:
await backend.close()
# ============================================
# QUEUE MANAGEMENT
# ============================================
def populate_queue(db: Session, dates: List[date], priorities: Dict[date, int] = None):
"""
Add dates to the scrape queue, skipping any already pending/processing.
Args:
db: Database session
dates: Dates to add to the queue
priorities: Optional priority map (higher = scraped first). Default: 0
"""
if not dates:
return 0
added = 0
for rate_date in dates:
priority = (priorities or {}).get(rate_date, 0)
try:
db.execute(
text("""
INSERT INTO booking_scrape_queue (rate_date, status, priority)
VALUES (:rate_date, 'pending', :priority)
ON CONFLICT (rate_date, status) DO UPDATE SET
priority = GREATEST(booking_scrape_queue.priority, :priority)
"""),
{'rate_date': rate_date, 'priority': priority}
)
added += 1
except Exception:
# Ignore duplicates or constraint issues
pass
db.commit()
logger.info(f"Queue: added/updated {added} dates")
return added
def get_pending_queue_items(db: Session, limit: int = 50) -> List[Dict[str, Any]]:
"""Get pending queue items ordered by priority (highest first), then date."""
result = db.execute(
text("""
SELECT id, rate_date, priority, attempts, max_attempts
FROM booking_scrape_queue
WHERE status = 'pending' AND attempts < max_attempts
ORDER BY priority DESC, rate_date ASC
LIMIT :limit
"""),
{'limit': limit}
)
return [dict(row._mapping) for row in result.fetchall()]
def mark_queue_item(db: Session, queue_id: int, status: str, error_message: str = None):
"""Update a queue item's status."""
if status == 'completed':
db.execute(
text("""
UPDATE booking_scrape_queue SET
status = 'completed',
completed_at = NOW(),
last_attempt_at = NOW(),
attempts = attempts + 1
WHERE id = :id
"""),
{'id': queue_id}
)
elif status == 'failed':
db.execute(
text("""
UPDATE booking_scrape_queue SET
status = CASE
WHEN attempts + 1 >= max_attempts THEN 'failed'
ELSE 'pending'
END,
last_attempt_at = NOW(),
attempts = attempts + 1,
error_message = :error
WHERE id = :id
"""),
{'id': queue_id, 'error': error_message}
)
db.commit()
def clear_old_queue_items(db: Session, days: int = 7):
"""Remove completed/failed queue items older than N days."""
db.execute(
text("""
DELETE FROM booking_scrape_queue
WHERE status IN ('completed', 'failed')
AND created_at < NOW() - INTERVAL ':days days'
""".replace(':days', str(int(days))))
)
db.commit()
async def process_queue(db: Session) -> Dict[str, Any]:
"""
Process pending items from the scrape queue.
Picks up pending items in priority order, scrapes each date,
and handles blocking/retries.
Returns:
Dict with processing results
"""
# Check if paused
if await is_scraper_paused(db):
return {
'success': False,
'error': 'Scraper is currently paused due to blocking.',
}
# Get config
config = get_scrape_config(db)
if not config:
return {
'success': False,
'error': 'No scrape location configured.',
}
# Get pending items
items = get_pending_queue_items(db, limit=200)
if not items:
return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'}
# Create batch
batch_id = create_scrape_batch(db, 'scheduled')
# Update batch with queue count
db.execute(
text("UPDATE booking_scrape_log SET dates_queued = :count WHERE batch_id = :bid"),
{'count': len(items), 'bid': str(batch_id)}
)
db.commit()
# Get backend
backend = get_scraper_backend(db)
total_hotels = 0
total_rates = 0
dates_completed = 0
dates_failed = 0
try:
for item in items:
rate_date = item['rate_date']
queue_id = item['id']
logger.info(f"Queue processing: {rate_date} (priority={item['priority']}, attempt={item['attempts']+1})")
result = await scrape_date(db, rate_date, backend, config, batch_id)
if result['blocked']:
# Mark this item as failed, pause, and stop
mark_queue_item(db, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
await set_scraper_paused(db, True, hours=2)
update_scrape_batch(
db, batch_id,
status='blocked',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
blocked=True
)
# Update dates counters
db.execute(
text("""
UPDATE booking_scrape_log SET
dates_completed = :completed,
dates_failed = :failed
WHERE batch_id = :bid
"""),
{'completed': dates_completed, 'failed': dates_failed + 1, 'bid': str(batch_id)}
)
db.commit()
return {
'success': False,
'blocked': True,
'block_reason': result.get('block_reason'),
'dates_completed': dates_completed,
'dates_failed': dates_failed + 1,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
if result['success']:
mark_queue_item(db, queue_id, 'completed')
total_hotels += result['hotels_count']
total_rates += result['rates_count']
dates_completed += 1
else:
mark_queue_item(db, queue_id, 'failed', result.get('error'))
dates_failed += 1
logger.warning(f"Queue: failed to scrape {rate_date}: {result.get('error')}")
# Update batch as completed
update_scrape_batch(
db, batch_id,
status='completed',
hotels_found=total_hotels,
rates_scraped=total_rates
)
db.execute(
text("""
UPDATE booking_scrape_log SET
dates_completed = :completed,
dates_failed = :failed
WHERE batch_id = :bid
"""),
{'completed': dates_completed, 'failed': dates_failed, 'bid': str(batch_id)}
)
db.commit()
return {
'success': True,
'blocked': False,
'dates_completed': dates_completed,
'dates_failed': dates_failed,
'hotels_found': total_hotels,
'rates_scraped': total_rates,
}
except Exception as e:
logger.error(f"Queue processing error: {e}")
update_scrape_batch(
db, batch_id,
status='failed',
hotels_found=total_hotels,
rates_scraped=total_rates,
error_message=str(e)
)
return {
'success': False,
'error': str(e),
'dates_completed': dates_completed,
'dates_failed': dates_failed,
}
finally:
await backend.close()
def get_competitor_matrix(
db: Session,
from_date: date,
to_date: date,
include_market: bool = False
) -> List[Dict[str, Any]]:
"""
Get rate comparison matrix for competitors.
Args:
db: Database session
from_date: Start date
to_date: End date
include_market: Include market tier hotels
Returns:
List of rate records for matrix display
"""
tier_filter = "h.tier IN ('own', 'competitor')"
if include_market:
tier_filter = "h.tier IN ('own', 'competitor', 'market')"
result = db.execute(
text(f"""
SELECT
r.rate_date,
h.id AS hotel_id,
h.name AS hotel_name,
h.tier,
h.display_order,
h.star_rating,
h.review_score,
r.availability_status,
r.rate_gross,
r.room_type,
r.breakfast_included,
r.free_cancellation,
r.no_prepayment,
r.rooms_left,
r.scraped_at
FROM booking_latest_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE {tier_filter}
AND h.is_active = TRUE
AND r.rate_date BETWEEN :from_date AND :to_date
ORDER BY r.rate_date, h.display_order, h.name
"""),
{'from_date': from_date, 'to_date': to_date}
)
return [dict(row._mapping) for row in result.fetchall()]
def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
"""
Get list of discovered hotels.
Args:
db: Database session
tier: Filter by tier ('own', 'competitor', 'market') or None for all
Returns:
List of hotel records
"""
where_clause = "WHERE is_active = TRUE"
if tier:
where_clause += f" AND tier = '{tier}'"
result = db.execute(
text(f"""
SELECT
id, booking_com_id, name, booking_com_url,
star_rating, review_score, review_count,
tier, display_order, notes,
first_seen_at, last_seen_at
FROM booking_com_hotels
{where_clause}
ORDER BY display_order, name
""")
)
return [dict(row._mapping) for row in result.fetchall()]
def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
"""Update a hotel's tier and display order."""
if tier not in ('own', 'competitor', 'market'):
raise ValueError(f"Invalid tier: {tier}")
params = {'hotel_id': hotel_id, 'tier': tier}
set_clause = "tier = :tier"
if display_order is not None:
set_clause += ", display_order = :order"
params['order'] = display_order
db.execute(
text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
params
)
db.commit()

100
backend/services/central_settings.py Normal file
View file

@ -0,0 +1,100 @@
"""
Client for the stack's central Settings service.
NewBook credentials are managed once in the Settings app (LXC 116) and
fetched live by every app — the same pattern as cashup / room-planner /
maintenance (see their lib/newbook.js). Falls back to None if the service
is unreachable so callers can fall back to app-local config.
"""
import logging
import os
import time
from typing import Optional
import httpx
logger = logging.getLogger(__name__)
SETTINGS_URL = os.getenv("SETTINGS_URL", "")
SETTINGS_SECRET = os.getenv("SETTINGS_SECRET", "")
_CACHE_TTL = 60 # seconds — credentials change rarely; avoid hammering the service
_cache: dict = {}
async def get_integration(name: str) -> Optional[dict]:
"""
Fetch integration config (e.g. 'newbook') from the central Settings
service. Returns the config dict, or None if unavailable/unconfigured.
"""
if not SETTINGS_URL or not SETTINGS_SECRET:
return None
cached = _cache.get(name)
if cached and (time.monotonic() - cached[0]) < _CACHE_TTL:
return cached[1]
url = f"{SETTINGS_URL}/settings/api/internal/integration/{name}"
try:
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(
url, headers={"Authorization": f"Bearer {SETTINGS_SECRET}"}
)
resp.raise_for_status()
data = resp.json()
_cache[name] = (time.monotonic(), data)
return data
except Exception as e:
logger.warning(f"Central settings fetch failed for '{name}': {e}")
return None
def _extract_newbook(s: Optional[dict]) -> Optional[dict]:
if not s:
return None
creds = {
"api_key": s.get("api_key") or "",
"username": s.get("username") or "",
"password": s.get("password") or "",
"region": s.get("region") or "eu",
}
# Only usable if the essential fields are present
if not (creds["api_key"] and creds["username"] and creds["password"]):
return None
return creds
async def get_newbook_credentials() -> Optional[dict]:
"""
Returns {'api_key', 'username', 'password', 'region'} from central
settings, or None if not available (caller should fall back).
"""
return _extract_newbook(await get_integration("newbook"))
def get_integration_sync(name: str) -> Optional[dict]:
"""Blocking variant of get_integration for sync job contexts."""
if not SETTINGS_URL or not SETTINGS_SECRET:
return None
cached = _cache.get(name)
if cached and (time.monotonic() - cached[0]) < _CACHE_TTL:
return cached[1]
url = f"{SETTINGS_URL}/settings/api/internal/integration/{name}"
try:
resp = httpx.get(
url, headers={"Authorization": f"Bearer {SETTINGS_SECRET}"}, timeout=5.0
)
resp.raise_for_status()
data = resp.json()
_cache[name] = (time.monotonic(), data)
return data
except Exception as e:
logger.warning(f"Central settings fetch failed for '{name}': {e}")
return None
def get_newbook_credentials_sync() -> Optional[dict]:
"""Blocking variant of get_newbook_credentials for sync job contexts."""
return _extract_newbook(get_integration_sync("newbook"))

28
backend/services/direct_profiles/__init__.py Normal file
View file

@ -0,0 +1,28 @@
from .guestline import GuestlineProfile
from .newbook_scrape import NewbookScrapeProfile
from .travelclick import TravelClickProfile
from .directbook import DirectBookProfile
from .base import BaseProfile
PROFILES = {
"guestline": GuestlineProfile,
"newbook_scrape": NewbookScrapeProfile,
"travelclick": TravelClickProfile,
"directbook": DirectBookProfile,
}
def get_profile(name: str) -> BaseProfile:
cls = PROFILES.get(name)
if not cls:
raise ValueError(f"Unknown engine profile: {name}")
return cls()
def detect_profile(url: str) -> dict | None:
"""Given a booking URL, return suggested profile name and extracted params."""
for name, cls in PROFILES.items():
result = cls.detect(url)
if result is not None:
return {"profile": name, **result}
return None

25
backend/services/direct_profiles/base.py Normal file
View file

@ -0,0 +1,25 @@
from abc import ABC, abstractmethod
class BaseProfile(ABC):
name: str = ""
label: str = ""
# Fields required to configure this engine, shown in the add-hotel form
# Each entry: {"key": "hotel_id", "label": "Hotel ID", "help": "e.g. THREEWAYS"}
required_params: list[dict] = []
@classmethod
@abstractmethod
def detect(cls, url: str) -> dict | None:
"""Return extracted params dict if URL matches this engine, else None."""
@abstractmethod
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
"""Return list of bookable arrival date strings YYYY-MM-DD."""
@abstractmethod
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
"""Return list of room/rate dicts for the stay.
Each dict must have: roomId, rateId, availability, prices[{amountBeforeTax, amountAfterTax}], currencyCode
prices list has one entry per night when nights > 1.
"""

175
backend/services/direct_profiles/directbook.py Normal file
View file

@ -0,0 +1,175 @@
import json
import re
from datetime import date, timedelta
from urllib.parse import quote
from .base import BaseProfile
API_BASE = "https://direct-book.com"
SETTINGS_HASH = "52d786f5c45232c8c16022bc3af6dab2e1994f4919b953afafe188095125e9b6"
QUOTESETS_HASH = "1012a6203854357e44786380240eefad6b2ad863aee6ba79748c81a851f29217"
ROOMTYPES_HASH = "8021345a2e1f993717b1960097489b456a6b2dc136990b6f919adba1b1fe2c1f"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
# Required to bypass Apollo CSRF protection on the /api/graphql endpoint
"Content-Type": "application/json",
}
def _graphql_url(operation: str, variables: dict, sha256: str) -> str:
return (
f"{API_BASE}/api/graphql"
f"?operationName={operation}"
f"&variables={_json_compact(variables)}"
f"&extensions={_json_compact({'persistedQuery': {'version': 1, 'sha256Hash': sha256}})}"
)
def _json_compact(obj) -> str:
return quote(json.dumps(obj, separators=(',', ':')), safe='')
async def _get_property_id(client, channel_code: str) -> str:
"""Fetch numeric propertyId from settings query."""
url = _graphql_url("settings", {"channelCode": channel_code}, SETTINGS_HASH)
r = await client.get(url, headers=HEADERS, timeout=20)
r.raise_for_status()
return str(r.json()["data"]["settings"]["uuid"])
class DirectBookProfile(BaseProfile):
name = "directbook"
label = "SiteMinder Direct Book"
required_params = [
{"key": "channel_code", "label": "Channel Code",
"help": "The property slug in the booking URL, e.g. 'grapevinestowdirect'"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
m = re.search(r'direct-book\.com/properties/([^/?#\s]+)', url)
if m:
return {"channel_code": m.group(1)}
return None
async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]:
channel_code = params["channel_code"]
url = _graphql_url(
"roomTypes",
{"channelCode": channel_code, "checkInDate": date.today().isoformat(),
"checkOutDate": (date.today() + timedelta(days=1)).isoformat(), "locale": "en"},
ROOMTYPES_HASH,
)
r = await client.get(url, headers=HEADERS, timeout=30)
r.raise_for_status()
room_types = r.json()["data"]["roomTypes"]
room_labels = {rt["uuid"]: rt["name"] for rt in room_types}
rate_labels = {}
for rt in room_types:
for rate in rt.get("rates", []):
rate_labels[rate["uuid"]] = rate["name"]
return room_labels, rate_labels
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
channel_code = params["channel_code"]
today = date.today()
# Fetch 12 months of availability in monthly chunks (API seems to accept wide ranges too)
end = today.replace(day=1) + timedelta(days=365)
# Build monthly windows to stay within API limits
available_dates: list[str] = []
current = today.replace(day=1)
while current <= end:
# Last day of the month
if current.month == 12:
month_end = current.replace(year=current.year + 1, month=1, day=1) - timedelta(days=1)
else:
month_end = current.replace(month=current.month + 1, day=1) - timedelta(days=1)
check_from = max(today, current).isoformat()
check_to = month_end.isoformat()
url = (
f"{API_BASE}/api/properties/{channel_code}/availability"
f"?checkInsFrom={check_from}&checkInsTo={check_to}"
)
try:
r = await client.get(url, headers=HEADERS, timeout=20)
r.raise_for_status()
for entry in r.json().get("result", []):
if entry.get("canCheckIn"):
d = entry["date"][:10]
if d >= today.isoformat():
available_dates.append(d)
except Exception:
pass
# Advance to next month
if current.month == 12:
current = current.replace(year=current.year + 1, month=1)
else:
current = current.replace(month=current.month + 1)
return sorted(set(available_dates))
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
channel_code = params["channel_code"]
# Get propertyId (numeric) — cache it on the params dict to avoid repeat fetches
if "property_id" not in params:
params["property_id"] = await _get_property_id(client, channel_code)
property_id = int(params["property_id"])
departure = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
url = _graphql_url(
"quoteSets",
{
"propertyId": property_id,
"promocode": "",
"checkInDate": arrival,
"checkOutDate": departure,
"adults": 2,
"children": 0,
"infants": 0,
"currencyFrom": "GBP",
"currencyTo": "GBP",
},
QUOTESETS_HASH,
)
r = await client.get(url, headers=HEADERS, timeout=20)
if r.status_code == 404:
return []
r.raise_for_status()
rooms = []
for qs in r.json()["data"].get("quoteSets", []):
room_id = str(qs["roomTypeId"])
for quote in qs.get("quotes", []):
rate_id = str(quote["roomRateId"])
price = quote["price"]["amount"]
available = quote.get("available", 0)
# Build per-night prices from breakdown if multi-night
breakdown = quote.get("breakdown", [])
if breakdown:
prices = [
{"amountBeforeTax": b["price"]["amount"], "amountAfterTax": b["price"]["amount"]}
for b in breakdown
]
else:
prices = [{"amountBeforeTax": price, "amountAfterTax": price}]
rooms.append({
"roomId": room_id,
"rateId": rate_id,
"availability": available,
"prices": prices,
"min_stay_nights": None,
"currencyCode": "GBP",
})
return rooms

49
backend/services/direct_profiles/guestline.py Normal file
View file

@ -0,0 +1,49 @@
import re
from datetime import date, timedelta
from .base import BaseProfile
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
}
class GuestlineProfile(BaseProfile):
name = "guestline"
label = "Guestline"
required_params = [
{"key": "hotel_id", "label": "Hotel ID", "help": "e.g. THREEWAYS — from the booking URL ?hotel= parameter"},
{"key": "collection_id", "label": "Collection ID", "help": "e.g. MT — the path segment before /availability in the booking URL"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
# Matches: https://booking.eu.guestline.app/MT/availability?hotel=THREEWAYS
m = re.search(r'booking\.(?:eu\.)?guestline\.app/([^/?\s]+)/availability\?hotel=([^&\s]+)', url)
if m:
return {"collection_id": m.group(1), "hotel_id": m.group(2)}
return None
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
hotel_id = params["hotel_id"]
today = date.today()
url = f"https://booking.eu.guestline.app/api/availabilities/{hotel_id}/arrivals"
r = await client.get(url, params={
"month": today.month, "year": today.year,
"adults": 2, "children": 0, "count": 12,
}, headers=HEADERS, timeout=20)
r.raise_for_status()
return [a["date"] for a in r.json().get("arrivals", [])]
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
hotel_id = params["hotel_id"]
collection_id = params.get("collection_id", "MT")
dep = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}/enhanced"
r = await client.get(url, params={
"arrival": arrival, "departure": dep, "adults": 2, "children": 0,
}, headers=HEADERS, timeout=20)
if r.status_code == 404:
return []
r.raise_for_status()
return r.json().get("availabilities", {}).get("rooms", [])

235
backend/services/direct_profiles/newbook_scrape.py Normal file
View file

@ -0,0 +1,235 @@
import json
import re
from datetime import date, timedelta
from .base import BaseProfile
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "*/*",
"X-Requested-With": "XMLHttpRequest",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
}
BASE_URL = "https://bookingseu.newbook.cloud"
def _base_params(slug: str, arrival: str, departure: str, nights: int) -> dict:
return {
"REMOTE_ADDR": "1.1.1.1",
"force_booking_channel_id": "",
"HTTP_REFERER": f"bookingseu.newbook.cloud/{slug}/index.php",
"discount_total_display": "0",
"force_category_id[]": "uK0ip@c7ty%8bQ#2i",
"force_category_type_id[]": "uK0ip@c7ty%8bQ#2i",
"no_billing_booking": "0",
"force_tariff_type_id[]": "uK0ip@c7ty%8bQ#2i",
"discount_id": "null",
"facebook_user_id": "",
"category_type_id": "",
"owner_occupied_booking_id": "",
"discount_code": "",
"booking_action": "",
"available_from": arrival,
"available_to": departure,
"nights": str(nights),
"adults": "2",
"children": "0",
"infants": "0",
"promo_code": "",
"language": "EN",
}
def _fmt_date(d: date) -> str:
"""Format date as NewBook expects: 'Mon 4 Jul 2026'"""
return d.strftime("%a %-d %b %Y")
def _parse_chart_html(html: str) -> tuple[dict[str, float], dict[str, str], dict[str, str], list[dict]]:
"""
Parse an availability_chart_responsive HTML response.
Returns:
counts: {cat_id: float} — room counts from category_sites_available JS var
cat_names: {cat_id: str} — friendly category names e.g. "Executive Double"
rate_names: {rate_id: str} — friendly tariff names e.g. "DIRECT B&B FLEX"
rooms: list of room/rate dicts compatible with base scraper format
"""
# Room counts from embedded JS
counts: dict[str, float] = {}
m = re.search(r'category_sites_available\s*=\s*(\{[^;]+\})', html)
if m:
try:
counts = {k: float(v) for k, v in json.loads(m.group(1)).items()}
except Exception:
pass
cat_names: dict[str, str] = {}
rate_names: dict[str, str] = {}
rooms = []
# Split by category box: offset="{cat_id}"
cat_blocks = re.split(r'<div[^>]+class="[^"]*newbook_online_category_box[^"]*"[^>]+offset="(\d+)"', html)
# cat_blocks: [pre, cat_id, block, cat_id, block, ...]
i = 1
while i < len(cat_blocks) - 1:
cat_id = cat_blocks[i]
block = cat_blocks[i + 1]
i += 2
avail = counts.get(cat_id, 0.0)
# Category friendly name — from category_name attr on any book button in this block
# e.g. category_name='Standard' or category_name='Executive Double'
cn_m = re.search(r"category_name='([^']+)'", block)
if not cn_m:
# Fallback: <h3><a ...>Name</a></h3>
cn_m = re.search(r'<h3>[^<]*<a[^>]*>([^<]+)</a>', block)
if cn_m:
cat_names[cat_id] = cn_m.group(1).strip()
# Split on tariff row boundaries
tariff_rows = re.split(r'class="[^"]*newbook_online_categories_tariff_type_rows[^"]*"', block)
for row in tariff_rows[1:]:
# Rate label
name_m = re.search(r'newbook_online_categories_tariff_type_label[^>]*>(.*?)</div>', row, re.DOTALL)
rate_label = re.sub(r'<[^>]+>', '', name_m.group(1)).strip() if name_m else ""
# Price
price_m = re.search(r'newbook_online_from_price_text[^>]*>£([\d.]+)<', row)
price = float(price_m.group(1)) if price_m else None
# Internal tariff type ID (stable across dates)
tid_m = re.search(r'tariff_type_id="(\d+)"', row)
rate_id = tid_m.group(1) if tid_m else rate_label
# Store rate label for this rate_id
if rate_id and rate_label:
rate_names[rate_id] = rate_label
# Min-stay: requires_date_change class + optional extend_nights attr
# If extend_nights present: min_stay = 1 + N; if absent: default to 2
min_stay = None
if 'requires_date_change' in row:
en_m = re.search(r'extend_nights="(\d+)"', row)
min_stay = 1 + int(en_m.group(1)) if en_m else 2
if rate_label and price is not None:
rooms.append({
"roomId": cat_id,
"rateId": rate_id,
"rateLabel": rate_label,
"availability": int(avail),
"prices": [{"amountBeforeTax": price, "amountAfterTax": price}],
"min_stay_nights": min_stay,
"currencyCode": "GBP",
})
return counts, cat_names, rate_names, rooms
class NewbookScrapeProfile(BaseProfile):
name = "newbook_scrape"
label = "NewBook (HTML scrape)"
required_params = [
{"key": "slug", "label": "Property Slug",
"help": "The path segment in the booking URL, e.g. 'numberfour' from bookingseu.newbook.cloud/numberfour/"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
m = re.search(r'bookingseu\.newbook\.cloud/([^/?#\s]+)', url)
if m and m.group(1) not in ('index.php',):
return {"slug": m.group(1)}
return None
async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]:
"""
Return ({cat_id: friendly_name}, {rate_id: friendly_name}) from a single chart call.
Used by discovery to pre-populate room_labels and rate_labels.
"""
slug = params["slug"]
today = date.today()
base = _base_params(slug, _fmt_date(today), _fmt_date(today + timedelta(days=1)), 1)
r = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive",
data=base, headers=HEADERS, timeout=30,
)
r.raise_for_status()
_, cat_names, rate_names, _ = _parse_chart_html(r.text)
return cat_names, rate_names
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
"""
Use the calendar endpoint to collect all available arrival dates
across all room types. One call per room type, union of available dates.
We first do a chart call to discover category IDs, then calendar per category.
"""
slug = params["slug"]
today = date.today()
arrival_str = _fmt_date(today)
departure_str = _fmt_date(today + timedelta(days=1))
# Step 1: chart call to discover category IDs and names
base = _base_params(slug, arrival_str, departure_str, 1)
r = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive",
data=base, headers=HEADERS, timeout=30,
)
r.raise_for_status()
html = r.text
cat_ids = re.findall(r'class="[^"]*newbook_online_category_box[^"]*"[^>]+offset="(\d+)"', html)
if not cat_ids:
return []
# Step 2: calendar call per category, collect available dates
available_dates: set[str] = set()
more_tariffs = {f"more_tariffs_{cid}": "1" for cid in cat_ids}
for cat_id in cat_ids:
cal_params = {
**base,
**more_tariffs,
"query": "newbook_calendar_initialise",
"calendar_category_id": cat_id,
}
try:
cr = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=data",
data=cal_params, headers=HEADERS, timeout=30,
)
cr.raise_for_status()
cal_data = cr.json()
cal_html = cal_data.get("calendar_display", "")
for dm in re.finditer(r'class="day available[^"]*"\s+data-date="(\d{4}-\d{2}-\d{2})"', cal_html):
available_dates.add(dm.group(1))
except Exception:
pass
return sorted(d for d in available_dates if d >= today.isoformat())
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
"""
POST to availability_chart_responsive for a specific date.
Returns list of room/rate dicts compatible with the base scraper format.
Also injects min_stay_nights onto each row.
"""
slug = params["slug"]
arr = date.fromisoformat(arrival)
dep = arr + timedelta(days=nights)
arr_str = _fmt_date(arr)
dep_str = _fmt_date(dep)
body = _base_params(slug, arr_str, dep_str, nights)
r = await client.post(
f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive",
data=body, headers=HEADERS, timeout=30,
)
if r.status_code == 404:
return []
r.raise_for_status()
_, _, _, rooms = _parse_chart_html(r.text)
return rooms

166
backend/services/direct_profiles/travelclick.py Normal file
View file

@ -0,0 +1,166 @@
import re
import httpx
from datetime import date, timedelta
from .base import BaseProfile
API_BASE = "https://api.travelclick.com"
TOKEN_URL = f"{API_BASE}/oauth/token-referer?grant_type=client_credentials"
HEADERS_BASE = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json",
"Content-Type": "application/json",
}
async def _get_token(client, referer: str) -> str:
r = await client.post(TOKEN_URL, headers={**HEADERS_BASE, "Referer": referer}, timeout=20)
r.raise_for_status()
return r.json()["access_token"]
def _auth_headers(token: str, referer: str) -> dict:
return {**HEADERS_BASE, "Authorization": f"Bearer {token}", "Referer": referer}
class TravelClickProfile(BaseProfile):
name = "travelclick"
label = "TravelClick / Amadeus"
required_params = [
{"key": "hotel_code", "label": "Hotel Code",
"help": "Numeric hotel ID, e.g. 77346 — visible in the booking engine network requests"},
{"key": "booking_url", "label": "Booking URL Base",
"help": "e.g. https://reservations.bespokehotels.com/noelarmshotel/book/dates-of-stay"},
]
@classmethod
def detect(cls, url: str) -> dict | None:
m = re.search(r'(https://reservations\.bespokehotels\.com/[^/]+/book/[^?#\s]+)', url)
if not m:
return None
booking_url = m.group(1)
# hotel_code is in inline JS as bookingEngineHotelId: '77346' on the booking page
hotel_code = ""
try:
r = httpx.get(booking_url, timeout=10, follow_redirects=True,
headers={"User-Agent": "Mozilla/5.0"})
hm = re.search(r'bookingEngineHotelId\s*:\s*[\'"](\d+)[\'"]', r.text)
if hm:
hotel_code = hm.group(1)
except Exception:
pass
return {"booking_url": booking_url, "hotel_code": hotel_code}
async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]:
hotel_code = params["hotel_code"]
referer = params["booking_url"]
token = await _get_token(client, referer)
r = await client.get(
f"{API_BASE}/be5-entity/v2/hotels/{hotel_code}/content",
params={"include": "roomtypes,rateplans", "lang": "EN_US"},
headers=_auth_headers(token, referer),
timeout=30,
)
r.raise_for_status()
data = r.json()
# roomtypes[].roomTypeId (numeric, used as roomId in avail) -> roomTypeName
room_labels = {
str(rt["roomTypeId"]): rt.get("roomTypeName", str(rt["roomTypeId"]))
for rt in data.get("roomtypes", [])
}
# ratePlans[].rateplanId (numeric, used as rateId in avail) -> rateplanName
rate_labels = {
str(rp["rateplanId"]): rp.get("rateplanName", str(rp["rateplanId"]))
for rp in data.get("ratePlans", [])
}
return room_labels, rate_labels
async def fetch_arrival_dates(self, client, params: dict) -> list[str]:
hotel_code = params["hotel_code"]
referer = params["booking_url"]
token = await _get_token(client, referer)
today = date.today()
end = today + timedelta(days=365)
body = {
"hotelCode": int(hotel_code),
"currency": "GBP",
"lang": "EN_US",
"dateIn": today.isoformat(),
"dateOut": end.isoformat(),
"multiRoomOccupancy": [{"adults": 2, "infant": 0, "children": 0}],
"bookerIdentifier": "",
"partnerIdentifier": "",
}
r = await client.post(
f"{API_BASE}/be5-shop/v1/hotel/{hotel_code}/basicavail/multi-room",
json=body,
headers=_auth_headers(token, referer),
timeout=30,
)
r.raise_for_status()
data = r.json()
return [
d["date"]
for d in data.get("dates", [])
if d.get("isAvailable") and d["date"] >= today.isoformat()
]
async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]:
hotel_code = params["hotel_code"]
referer = params["booking_url"]
token = await _get_token(client, referer)
departure = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
body = {
"roomStay": {
"startDate": arrival,
"endDate": departure,
"guestCount": {"adults": 2, "infants": 0, "children": {"ages": None, "count": 0}},
"roomQuantity": 1,
"productSearchCriteria": {
"sortingPreference": "SORT_BY_ORDER",
"includeUnavailable": False,
},
},
"languageCode": "EN_US",
"disableLocaitonSharing": False,
"currencyCode": "GBP",
"tpaExtension": [],
"includeMemberRate": True,
"includeNightlyRates": True,
}
r = await client.post(
f"{API_BASE}/be5-shop/v2/hotels/{hotel_code}/avail",
json=body,
headers=_auth_headers(token, referer),
timeout=30,
)
if r.status_code == 404:
return []
r.raise_for_status()
data = r.json()
# Response: roomStays[0].roomtypes[].products[] (both Regular rates and Packages)
room_stay = (data.get("roomStays") or [{}])[0]
rooms = []
for rt in room_stay.get("roomtypes", []):
room_id = str(rt["roomtypeId"])
for product in rt.get("products", []):
rate_id = str(product["productId"])
nightly = product.get("nightlyRates", [])
if not nightly:
continue
avail = nightly[0].get("inventoryCount", 0)
prices = [
{"amountBeforeTax": n["amountBeforeTax"], "amountAfterTax": n.get("amountTotal", n["amountBeforeTax"])}
for n in nightly
]
rooms.append({
"roomId": room_id,
"rateId": rate_id,
"availability": avail,
"prices": prices,
"min_stay_nights": None, # min-stay comes from basicavail per date
"currencyCode": "GBP",
})
return rooms

295
backend/services/direct_scraper.py Normal file
View file

@ -0,0 +1,295 @@
"""
Direct booking engine scraper — adapted from guestline-monitor/app/scraper.py.
Replaces SQLite per-hotel DBs with PostgreSQL via SyncSessionLocal.
Logic (min-stay detection, discovery date sampling) is unchanged from the original.
"""
import asyncio
import logging
from datetime import datetime, timezone, date, timedelta
import httpx
from sqlalchemy import text
from database import SyncSessionLocal
from services.direct_profiles import get_profile
log = logging.getLogger(__name__)
REQUEST_DELAY = 10.0
DISCOVERY_DELAY = 2.0
# Track running discovery scrapes: hotel_id -> status dict
_discovery_status: dict[int, dict] = {}
def _discovery_dates() -> list[str]:
"""42 spread dates: one per DOW over 6 months, 6 weeks apart."""
dates = []
today = date.today()
for week_offset in range(6):
base = today + timedelta(weeks=week_offset * 4)
for dow in range(7):
days_ahead = (dow - base.weekday()) % 7
d = base + timedelta(days=days_ahead + 7)
iso = d.isoformat()
if iso not in dates:
dates.append(iso)
return sorted(dates)
async def run_discovery(hotel_id: int, profile_name: str, params: dict):
"""Sample ~42 spread dates to find all room/rate type IDs for a competitor hotel."""
_discovery_status[hotel_id] = {
"state": "running", "done": 0, "total": 0,
"found_rooms": [], "found_rates": [],
}
profile = get_profile(profile_name)
dates = _discovery_dates()
_discovery_status[hotel_id]["total"] = len(dates)
found_rooms: set[str] = set()
found_rates: set[str] = set()
async with httpx.AsyncClient() as client:
for i, arrival in enumerate(dates):
await asyncio.sleep(DISCOVERY_DELAY)
try:
rooms = await profile.fetch_night_rates(client, params, arrival)
for room in rooms:
found_rooms.add(room["roomId"])
found_rates.add(room["rateId"])
except Exception as e:
log.warning(f"Discovery hotel {hotel_id} {arrival}: {e}")
_discovery_status[hotel_id]["done"] = i + 1
_discovery_status[hotel_id]["found_rooms"] = sorted(found_rooms)
_discovery_status[hotel_id]["found_rates"] = sorted(found_rates)
# Fetch friendly names if the profile supports it
if hasattr(profile, "fetch_category_names"):
try:
cat_names, rate_names = await profile.fetch_category_names(client, params)
_discovery_status[hotel_id]["room_labels"] = cat_names
_discovery_status[hotel_id]["rate_labels"] = rate_names
# Merge into DB (existing user labels win)
db = SyncSessionLocal()
try:
row = db.execute(
text("SELECT room_labels, rate_labels FROM direct_competitor_hotels WHERE id = :id"),
{"id": hotel_id}
).mappings().fetchone()
if row:
import json
existing_rooms = row["room_labels"] or {}
existing_rates = row["rate_labels"] or {}
merged_rooms = {**cat_names, **existing_rooms}
merged_rates = {**rate_names, **existing_rates}
db.execute(
text("""UPDATE direct_competitor_hotels
SET room_labels = :rl, rate_labels = :ratel
WHERE id = :id"""),
{"rl": json.dumps(merged_rooms), "ratel": json.dumps(merged_rates), "id": hotel_id}
)
db.commit()
finally:
db.close()
except Exception as e:
log.warning(f"Discovery hotel {hotel_id}: could not fetch category names: {e}")
_discovery_status[hotel_id]["state"] = "complete"
log.info(f"Discovery complete hotel {hotel_id}: {len(found_rooms)} rooms, {len(found_rates)} rates")
def get_discovery_status(hotel_id: int) -> dict:
return _discovery_status.get(hotel_id, {"state": "idle"})
def run_scrape(hotel_id: int, profile_name: str, params: dict):
"""Full scrape run for one configured hotel. Writes to direct_rates + direct_scrape_runs."""
scraped_at = datetime.now(timezone.utc)
log.info(f"Direct scrape started for hotel {hotel_id}")
profile = get_profile(profile_name)
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at))
finally:
loop.close()
db = SyncSessionLocal()
try:
db.execute(
text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"),
{"ts": scraped_at, "id": hotel_id}
)
db.commit()
finally:
db.close()
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
async with httpx.AsyncClient() as client:
try:
arrival_dates = await profile.fetch_arrival_dates(client, params)
except Exception as e:
log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}")
return
log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates")
db = SyncSessionLocal()
try:
result = db.execute(
text("INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found) VALUES (:hid, :ts, :df) RETURNING id"),
{"hid": hotel_id, "ts": scraped_at, "df": len(arrival_dates)}
)
run_id = result.fetchone()[0]
db.commit()
prev_dates = {r[0].isoformat() for r in db.execute(
text("SELECT DISTINCT stay_date FROM direct_rates WHERE hotel_id = :hid AND stay_date >= :today"),
{"hid": hotel_id, "today": date.today()}
).fetchall()}
finally:
db.close()
arrival_set = set(arrival_dates)
missing_dates = sorted(prev_dates - arrival_set)
if missing_dates:
log.info(f"Hotel {hotel_id}: {len(missing_dates)} dates absent, checking min-stay")
# Min-stay check for missing dates
for fd in missing_dates:
fd_date = date.fromisoformat(fd)
windows = [
(fd_date - timedelta(days=1), fd_date + timedelta(days=1)),
(fd_date, fd_date + timedelta(days=2)),
]
min_stay_rooms = None
min_stay_other_night = None
await asyncio.sleep(REQUEST_DELAY)
for win_start, win_end in windows:
try:
rooms_2n = await profile.fetch_night_rates(client, params, win_start.isoformat(), nights=2)
if rooms_2n:
min_stay_rooms = rooms_2n
companion = win_start if win_start.isoformat() != fd else (win_start + timedelta(days=1))
min_stay_other_night = companion.isoformat()
log.info(f" Hotel {hotel_id} {fd}: min-stay detected")
break
except Exception as e:
log.warning(f" Hotel {hotel_id} {fd}: min-stay check {win_start}: {e}")
db = SyncSessionLocal()
try:
last_rows = db.execute(
text("""SELECT DISTINCT ON (room_id, rate_id)
room_id, rate_id, availability, price_excl, price_incl, currency
FROM direct_rates
WHERE hotel_id = :hid AND stay_date = :fd
ORDER BY room_id, rate_id, scraped_at DESC"""),
{"hid": hotel_id, "fd": fd}
).mappings().fetchall()
insert_rows = []
for r in last_rows:
if min_stay_rooms is not None:
match = next(
(m for m in min_stay_rooms
if m["roomId"] == r["room_id"] and m["rateId"] == r["rate_id"]
and m.get("prices")),
None
)
if match:
prices = match["prices"]
win_start_used = windows[0][0] if min_stay_other_night == windows[0][0].isoformat() else windows[1][0]
idx = 1 if win_start_used.isoformat() != fd else 0
if len(prices) > idx:
p = prices[idx]
insert_rows.append({
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
"room_id": r["room_id"], "rate_id": r["rate_id"],
"avail": match.get("availability", r["availability"]),
"pe": p["amountBeforeTax"], "pi": p["amountAfterTax"],
"cur": match.get("currencyCode", r["currency"]), "ms": 2
})
continue
insert_rows.append({
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
"room_id": r["room_id"], "rate_id": r["rate_id"],
"avail": r["availability"], "pe": r["price_excl"], "pi": r["price_incl"],
"cur": r["currency"], "ms": 2
})
else:
insert_rows.append({
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
"room_id": r["room_id"], "rate_id": r["rate_id"],
"avail": 0, "pe": r["price_excl"], "pi": r["price_incl"],
"cur": r["currency"], "ms": None
})
if insert_rows:
db.execute(
text("""INSERT INTO direct_rates
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
availability, price_excl, price_incl, currency, min_stay_nights)
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
:avail, :pe, :pi, :cur, :ms)"""),
insert_rows
)
db.commit()
status = "min-stay(2N)" if min_stay_rooms else "fully-booked"
log.info(f" Hotel {hotel_id} {fd}: recorded as {status}")
finally:
db.close()
# Scrape all bookable dates
rows_saved = 0
for i, arrival in enumerate(arrival_dates):
await asyncio.sleep(REQUEST_DELAY)
try:
rooms = await profile.fetch_night_rates(client, params, arrival)
if not rooms:
continue
insert_rows = [
{
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": arrival,
"room_id": room["roomId"], "rate_id": room["rateId"],
"avail": room["availability"],
"pe": room["prices"][0]["amountBeforeTax"],
"pi": room["prices"][0]["amountAfterTax"],
"cur": room.get("currencyCode", "GBP"),
"ms": room.get("min_stay_nights")
}
for room in rooms if room.get("prices")
]
if insert_rows:
db = SyncSessionLocal()
try:
db.execute(
text("""INSERT INTO direct_rates
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
availability, price_excl, price_incl, currency, min_stay_nights)
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
:avail, :pe, :pi, :cur, :ms)"""),
insert_rows
)
db.commit()
finally:
db.close()
rows_saved += len(insert_rows)
log.info(f" Hotel {hotel_id} {arrival}: {len(rooms)} combos [{i+1}/{len(arrival_dates)}]")
except Exception as e:
log.error(f" Hotel {hotel_id} {arrival}: ERROR - {e}")
db = SyncSessionLocal()
try:
db.execute(
text("UPDATE direct_scrape_runs SET rows_saved = :rs WHERE id = :id"),
{"rs": rows_saved, "id": run_id}
)
db.commit()
finally:
db.close()
log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved")

863
backend/services/newbook_rates_client.py Normal file
View file

@ -0,0 +1,863 @@
"""
Newbook Rates Client
Fetches current rack rates from Newbook API for revenue forecasting.
Uses the bookings_availability_pricing endpoint to simulate booking requests.
This client is READ-ONLY - it only queries available rates, never creates bookings.
"""
import os
import httpx
import asyncio
import logging
from datetime import date, timedelta
from decimal import Decimal
from typing import Optional, List, Dict
logger = logging.getLogger(__name__)
class NewbookRatesError(Exception):
"""Custom exception for Newbook rates API errors"""
pass
class NewbookRatesClient:
"""
Async client for fetching current rates from Newbook API.
Uses bookings_availability_pricing endpoint which simulates a booking request.
Handles minimum stay restrictions by extending the stay period when needed.
Rate limiting: ~100 requests/min, using 0.75s delay between requests
"""
BASE_URL = "https://api.newbook.cloud/rest"
def __init__(self, api_key: str = None, username: str = None, password: str = None,
region: str = None, vat_rate: Decimal = Decimal('0.20')):
self.api_key = api_key or os.getenv("NEWBOOK_API_KEY")
self.username = username or os.getenv("NEWBOOK_USERNAME")
self.password = password or os.getenv("NEWBOOK_PASSWORD")
self.region = region or os.getenv("NEWBOOK_REGION")
self.vat_rate = vat_rate
if not all([self.api_key, self.username, self.password, self.region]):
logger.warning("Newbook credentials not fully configured")
def _get_url(self, endpoint: str) -> str:
"""Get full URL for an endpoint"""
return f"{self.BASE_URL}/{endpoint}"
@classmethod
async def from_db(cls, db):
"""
Create client with credentials from the central Settings service
(stack-wide NewBook config), falling back to the app-local
system_config table. VAT rate stays app-local either way.
"""
from sqlalchemy import text
from services.central_settings import get_newbook_credentials
result = await db.execute(
text("SELECT config_key, config_value FROM system_config WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')")
)
rows = result.fetchall()
config = {row.config_key: row.config_value for row in rows}
vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20'))
central = await get_newbook_credentials()
if central:
return cls(**central, vat_rate=vat_rate)
return cls(
api_key=config.get('newbook_api_key'),
username=config.get('newbook_username'),
password=config.get('newbook_password'),
region=config.get('newbook_region'),
vat_rate=vat_rate
)
async def __aenter__(self):
self.client = httpx.AsyncClient(timeout=60.0)
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.client.aclose()
def _get_auth_payload(self) -> dict:
"""Get base authentication payload"""
return {
"api_key": self.api_key,
"region": self.region
}
async def get_category_rates(
self,
category_id: str,
from_date: date,
to_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> List[Dict]:
"""
Fetch current rates for a category over a date range.
Uses daily=true to get per-night rates. Handles minimum stay
restrictions by extending the period when needed.
Args:
category_id: Newbook category ID
from_date: Start date for rates
to_date: End date for rates (inclusive)
guests_adults: Number of adult guests (default 2)
guests_children: Number of child guests (default 0)
Returns:
List of dicts with {date, gross_rate, net_rate}
"""
rates = []
current_date = from_date
while current_date <= to_date:
try:
# Fetch rates for up to 7 days at a time to optimize API calls
batch_end = min(current_date + timedelta(days=6), to_date)
batch_rates = await self._fetch_rates_batch(
category_id, current_date, batch_end, guests_adults, guests_children
)
rates.extend(batch_rates)
# Move to next batch
current_date = batch_end + timedelta(days=1)
except Exception as e:
logger.error(f"Failed to fetch rates for category {category_id} starting {current_date}: {e}")
# Skip this batch and continue
current_date = current_date + timedelta(days=7)
# Rate limiting - ALWAYS wait 1.5s between requests, even after errors
await asyncio.sleep(1.5)
return rates
async def get_single_night_rates(
self,
category_id: str,
from_date: date,
to_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> List[Dict]:
"""
Fetch rates with single-night queries for accurate per-day tariff availability.
Unlike get_category_rates which batches, this queries each date individually
as a 1-night stay. This gives accurate tariff_success per night, catching
issues like Valentine's Day blocking only that night, not a whole week.
Much slower but necessary for accurate bookability data.
Args:
category_id: Newbook category ID
from_date: Start date for rates
to_date: End date for rates (inclusive)
guests_adults: Number of adult guests (default 2)
guests_children: Number of child guests (default 0)
Returns:
List of dicts with {date, gross_rate, net_rate, tariffs_data}
"""
rates = []
current_date = from_date
while current_date <= to_date:
try:
# Single-night query for accurate tariff availability
batch_rates = await self._fetch_rates_batch(
category_id, current_date, current_date, guests_adults, guests_children
)
rates.extend(batch_rates)
except Exception as e:
logger.warning(f"Failed to fetch single-night rate for {category_id} on {current_date}: {e}")
# Continue with next date
current_date += timedelta(days=1)
# Rate limiting - wait between each single-night query
await asyncio.sleep(1.0)
return rates
async def fetch_single_date_all_categories(
self,
for_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[str, List[Dict]]:
"""
Fetch single-night rates for ALL categories for one date.
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
"""
return await self._fetch_all_categories_batch(
for_date, guests_adults, guests_children
)
async def fetch_multi_night_for_date(
self,
for_date: date,
nights: int,
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[str, Dict[str, bool]]:
"""
Fetch multi-night availability for ALL categories for one date.
Returns:
Dict of {category_id: {tariff_name: available}}
"""
return await self._fetch_all_categories_multi_night(
for_date, nights, guests_adults, guests_children
)
async def get_all_categories_single_night_rates(
self,
from_date: date,
to_date: date,
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[str, List[Dict]]:
"""
Fetch rates for ALL categories with single-night queries.
More efficient than get_single_night_rates - omits category_id to get
all categories in a single API call per date. This reduces API calls
from (categories × days) to just (days).
Args:
from_date: Start date for rates
to_date: End date for rates (inclusive)
guests_adults: Number of adult guests (default 2)
guests_children: Number of child guests (default 0)
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}, ...]}
"""
all_rates: Dict[str, List[Dict]] = {}
current_date = from_date
total_days = (to_date - from_date).days + 1
day_count = 0
while current_date <= to_date:
day_count += 1
try:
# Single-night query WITHOUT category_id - returns ALL categories
category_rates = await self._fetch_all_categories_batch(
current_date, guests_adults, guests_children
)
# Merge into all_rates dict
for cat_id, rates in category_rates.items():
if cat_id not in all_rates:
all_rates[cat_id] = []
all_rates[cat_id].extend(rates)
logger.info(f"Fetched {current_date} ({day_count}/{total_days}) - {len(category_rates)} categories")
except Exception as e:
logger.warning(f"Failed to fetch rates for {current_date}: {e}")
# Continue with next date
current_date += timedelta(days=1)
# Rate limiting - wait between each query
await asyncio.sleep(1.0)
return all_rates
async def _fetch_all_categories_batch(
self,
for_date: date,
guests_adults: int,
guests_children: int,
retry_count: int = 0
) -> Dict[str, List[Dict]]:
"""
Fetch rates for ALL categories for a single date.
Omits category_id from request - Newbook returns all available categories.
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
"""
# Single-night query
period_from = f"{for_date.isoformat()} 14:00:00"
period_to = f"{(for_date + timedelta(days=1)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"daily_mode": "true"
# NO category_id - returns all categories
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting with exponential backoff
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1)
logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3")
await asyncio.sleep(wait_time)
return await self._fetch_all_categories_batch(
for_date, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
raise NewbookRatesError(f"API returned failure: {data.get('message')}")
# Parse all categories from response
return self._parse_all_categories_tariffs(data, for_date)
async def _fetch_all_categories_multi_night(
self,
for_date: date,
nights: int,
guests_adults: int = 2,
guests_children: int = 0,
retry_count: int = 0
) -> Dict[str, Dict[str, bool]]:
"""
Fetch multi-night availability for ALL categories for a specific date.
Used to verify that rates with min_stay requirements are actually bookable.
Args:
for_date: Check-in date
nights: Number of nights to query (e.g., 2 for min_stay=2)
guests_adults: Number of adult guests
guests_children: Number of child guests
Returns:
Dict of {category_id: {tariff_name: available}}
"""
# Multi-night query
period_from = f"{for_date.isoformat()} 14:00:00"
period_to = f"{(for_date + timedelta(days=nights)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"daily_mode": "true"
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1)
logger.warning(f"Rate limited (multi-night), waiting {wait_time}s")
await asyncio.sleep(wait_time)
return await self._fetch_all_categories_multi_night(
for_date, nights, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
raise NewbookRatesError(f"API returned failure: {data.get('message')}")
# Parse availability by tariff name for each category
results: Dict[str, Dict[str, bool]] = {}
if not isinstance(data.get("data"), dict):
return results
for key, cat_data in data["data"].items():
if not (key.isdigit() or str(key).isnumeric()):
continue
if not isinstance(cat_data, dict):
continue
category_id = str(key)
tariffs_available = cat_data.get("tariffs_available", [])
results[category_id] = {}
for tariff in tariffs_available:
tariff_name = tariff.get("tariff_name", "")
tariff_label = tariff.get("tariff_label", "")
# Check tariff_success (API returns string "true"/"false")
tariff_success = str(tariff.get("tariff_success", False)).lower() in ("true", "1")
# Available if API says success, OR if rates are quoted and no restriction message
is_available = tariff_success or (
bool(tariff.get("tariffs_quoted")) and not tariff.get("tariff_message")
)
# Store under both tariff_name and tariff_label for flexible matching
results[category_id][tariff_name] = is_available
if tariff_label and tariff_label != tariff_name:
results[category_id][tariff_label] = is_available
return results
async def get_multi_night_availability(
self,
dates_by_nights: Dict[int, List[date]],
guests_adults: int = 2,
guests_children: int = 0
) -> Dict[date, Dict[str, Dict[str, bool]]]:
"""
Fetch multi-night availability for specific dates grouped by stay length.
Checks if a tariff is available when booking N nights starting from each date.
Args:
dates_by_nights: Dict of {nights: [dates]} e.g., {2: [date1, date2], 3: [date3]}
guests_adults: Number of adult guests
guests_children: Number of child guests
Returns:
Dict of {date: {category_id: {tariff_name: available}}}
"""
results: Dict[date, Dict[str, Dict[str, bool]]] = {}
total_queries = sum(len(dates) for dates in dates_by_nights.values())
query_count = 0
for nights, dates in dates_by_nights.items():
for query_date in dates:
query_count += 1
try:
result = await self._fetch_all_categories_multi_night(
query_date, nights, guests_adults, guests_children
)
results[query_date] = result
logger.info(f"Multi-night check {query_count}/{total_queries}: {query_date} ({nights} nights)")
except Exception as e:
logger.warning(f"Failed multi-night check for {query_date}: {e}")
# Rate limiting
await asyncio.sleep(1.0)
return results
async def _fetch_rates_batch(
self,
category_id: str,
from_date: date,
to_date: date,
guests_adults: int,
guests_children: int,
retry_count: int = 0
) -> List[Dict]:
"""
Fetch rates for a batch of dates (up to 7 days).
Handles minimum stay restrictions by extending the period and
extracting only the dates we need.
Returns:
List of dicts with {date, gross_rate, net_rate}
"""
# Format dates with times (check-in 14:00, check-out 10:00)
period_from = f"{from_date.isoformat()} 14:00:00"
period_to = f"{(to_date + timedelta(days=1)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"category_id": category_id,
"daily_mode": "true" # Get per-night breakdown
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting with exponential backoff
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1) # 60s, 120s, 180s
logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3")
await asyncio.sleep(wait_time)
return await self._fetch_rates_batch(
category_id, from_date, to_date, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
# Check if minimum stay restriction
categories = data.get("data", {}).get("categories", [])
if categories:
cat = categories[0] if isinstance(categories, list) else categories.get(category_id, {})
min_periods = cat.get("minimum_periods", 1)
if min_periods > 1:
# Extend the stay to meet minimum and retry
extended_to = from_date + timedelta(days=min_periods)
logger.info(f"Minimum stay {min_periods} nights for category {category_id}, extending to {extended_to}")
return await self._fetch_rates_with_min_stay(
category_id, from_date, to_date, extended_to,
guests_adults, guests_children
)
raise NewbookRatesError(f"API returned failure: {data.get('message')}")
# Parse tariffs_quoted from response
return self._parse_tariffs(data, from_date, to_date)
async def _fetch_rates_with_min_stay(
self,
category_id: str,
from_date: date,
to_date: date,
extended_to: date,
guests_adults: int,
guests_children: int,
retry_count: int = 0
) -> List[Dict]:
"""
Fetch rates with extended period for minimum stay requirement.
Args:
category_id: Newbook category ID
from_date: Original start date
to_date: Original end date (dates we want)
extended_to: Extended end date to meet minimum stay
guests_adults: Number of adults
guests_children: Number of children
Returns:
List of rates for the original date range only
"""
period_from = f"{from_date.isoformat()} 14:00:00"
period_to = f"{(extended_to + timedelta(days=1)).isoformat()} 10:00:00"
payload = self._get_auth_payload()
payload.update({
"period_from": period_from,
"period_to": period_to,
"adults": guests_adults,
"children": guests_children,
"infants": 0,
"category_id": category_id,
"daily_mode": "true"
})
response = await self.client.post(
self._get_url("bookings_availability_pricing"),
json=payload,
auth=(self.username, self.password)
)
# Handle rate limiting with exponential backoff
if response.status_code == 429:
if retry_count < 3:
wait_time = 60 * (retry_count + 1)
logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry")
await asyncio.sleep(wait_time)
return await self._fetch_rates_with_min_stay(
category_id, from_date, to_date, extended_to, guests_adults, guests_children, retry_count + 1
)
else:
raise NewbookRatesError(f"Rate limited after 3 retries")
if response.status_code != 200:
raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
data = response.json()
if not data.get("success"):
raise NewbookRatesError(f"API returned failure even with extended stay: {data.get('message')}")
# Parse tariffs but only return dates in our original range
return self._parse_tariffs(data, from_date, to_date)
def _parse_tariffs(self, data: dict, from_date: date, to_date: date) -> List[Dict]:
"""
Parse tariffs from API response.
With daily_mode=true, the API returns tariffs_quoted as a dict keyed by date.
Falls back to tariffs_available average if tariffs_quoted not available.
Args:
data: Full API response
from_date: Start date to include
to_date: End date to include
Returns:
List of dicts with {date, gross_rate, net_rate, tariffs_data}
tariffs_data contains all available tariff options for rate report
"""
rates = []
tariffs_quoted = {}
fallback_rate = None
inventory_items = []
all_tariffs_available = [] # Store all tariff options for reporting
# Find tariffs data in the response
if isinstance(data.get("data"), dict):
for key in data["data"].keys():
# Category IDs are numeric strings
if key.isdigit() or key.isnumeric():
cat_data = data["data"][key]
if isinstance(cat_data, dict):
tariffs_available = cat_data.get("tariffs_available", [])
all_tariffs_available = tariffs_available # Capture all options
if tariffs_available:
first_tariff = tariffs_available[0]
# tariffs_quoted is a dict keyed by date string
tariffs_quoted = first_tariff.get("tariffs_quoted", {})
# inventory_items are at tariff level (total for whole stay)
inventory_items = first_tariff.get("inventory_items", [])
# Fallback average rate
fallback_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0))
break
# If we have per-night tariffs_quoted dict, parse it
if isinstance(tariffs_quoted, dict) and tariffs_quoted:
num_nights = len(tariffs_quoted)
# Calculate per-night inventory item amount for items already included in tariff
included_inventory_per_night = Decimal('0')
for item in inventory_items:
already_included = item.get('amount_already_included_in_tariff_total', '')
if str(already_included).lower() == 'true':
total_amount = Decimal(str(item.get('amount', 0) or 0))
included_inventory_per_night += total_amount / num_nights
for date_str, tariff in tariffs_quoted.items():
try:
stay_date = date.fromisoformat(date_str)
except ValueError:
continue
# Only include dates in our range
if stay_date < from_date or stay_date > to_date:
continue
gross_rate = Decimal(str(tariff.get('amount', 0) or 0))
# Net = (gross - included_inventory_per_night) / (1 + VAT)
gross_after_inventory = gross_rate - included_inventory_per_night
net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01'))
# Build tariffs_data with day-specific rates
tariffs_data = self._build_tariffs_summary(all_tariffs_available, stay_date)
rates.append({
'date': stay_date,
'gross_rate': float(gross_rate),
'net_rate': float(net_rate),
'tariffs_data': tariffs_data
})
return rates
# Fallback: use average_nightly_tariff and apply to all dates
if fallback_rate and fallback_rate > 0:
net_rate = (fallback_rate / (1 + self.vat_rate)).quantize(Decimal('0.01'))
current_date = from_date
while current_date <= to_date:
# Build tariffs_data (no day-specific rates in fallback)
tariffs_data = self._build_tariffs_summary(all_tariffs_available, current_date)
rates.append({
'date': current_date,
'gross_rate': float(fallback_rate),
'net_rate': float(net_rate),
'tariffs_data': tariffs_data
})
current_date += timedelta(days=1)
return rates
logger.warning(f"No rate found in response for {from_date} to {to_date}")
return rates
def _build_tariffs_summary(self, tariffs_available: list, for_date: date = None) -> dict:
"""
Build a summary of all available tariff options for rate reporting.
Args:
tariffs_available: List of tariff dicts from API response
for_date: Optional specific date to extract day-specific rates
Returns:
Dict with tariff summaries - tariff_count and list of tariff details
"""
if not tariffs_available:
return {}
summary = {
'tariff_count': len(tariffs_available),
'tariffs': []
}
date_key = for_date.isoformat() if for_date else None
for idx, tariff in enumerate(tariffs_available):
# Get day-specific rate from tariffs_quoted if available
day_rate = None
if date_key:
tariffs_quoted = tariff.get('tariffs_quoted', {})
if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted:
day_quote = tariffs_quoted[date_key]
if isinstance(day_quote, dict):
day_rate = float(day_quote.get('amount', 0) or 0)
else:
day_rate = float(day_quote or 0)
# API uses tariff_label for the name
message = tariff.get('tariff_message', '')
# Extract minimum stay from message or dedicated field
min_stay = tariff.get('minimum_nights', None)
if min_stay is None and message:
# Try to parse from message like "Minimum 2 nights" or "2 Night Minimum"
import re
match = re.search(r'(\d+)\s*[Nn]ight\s*[Mm]inimum', message)
if not match:
match = re.search(r'[Mm]inimum\s+(\d+)\s*(?:night|period)', message)
if match:
min_stay = int(match.group(1))
# Extract advance booking requirement from message
min_advance_days = None
if message:
import re
advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
if advance_match:
min_advance_days = int(advance_match.group(1))
tariff_info = {
'name': tariff.get('tariff_label', 'Unknown'),
'description': tariff.get('tariff_short_description', ''),
'rate': day_rate, # Day-specific rate (None if not available)
'average_nightly': float(tariff.get('average_nightly_tariff', 0) or 0),
'success': str(tariff.get('tariff_success', False)).lower() in ('true', '1'),
'message': message,
'sort_order': idx, # Preserve Newbook ordering
'min_stay': min_stay, # Minimum nights required (if any)
'min_advance_days': min_advance_days, # Advance booking requirement (if any)
}
summary['tariffs'].append(tariff_info)
return summary
def _parse_all_categories_tariffs(self, data: dict, for_date: date) -> Dict[str, List[Dict]]:
"""
Parse tariffs from API response for ALL categories.
When category_id is omitted, data.data contains category IDs as keys,
each with their own tariffs_available.
Args:
data: Full API response
for_date: The date we queried
Returns:
Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
"""
results: Dict[str, List[Dict]] = {}
if not isinstance(data.get("data"), dict):
return results
for key, cat_data in data["data"].items():
# Category IDs are numeric strings like "1", "8", etc.
if not (key.isdigit() or str(key).isnumeric()):
continue
if not isinstance(cat_data, dict):
continue
category_id = str(key)
tariffs_available = cat_data.get("tariffs_available", [])
if not tariffs_available:
continue
# Get the first (best) tariff for gross/net calculation
first_tariff = tariffs_available[0]
tariffs_quoted = first_tariff.get("tariffs_quoted", {})
inventory_items = first_tariff.get("inventory_items", [])
# Get rate for this date
date_key = for_date.isoformat()
gross_rate = Decimal('0')
net_rate = Decimal('0')
if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted:
day_tariff = tariffs_quoted[date_key]
gross_rate = Decimal(str(day_tariff.get('amount', 0) or 0))
# Calculate included inventory per night
included_inventory = Decimal('0')
for item in inventory_items:
already_included = item.get('amount_already_included_in_tariff_total', '')
if str(already_included).lower() == 'true':
included_inventory += Decimal(str(item.get('amount', 0) or 0))
gross_after_inventory = gross_rate - included_inventory
net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01'))
else:
# Fallback to average
gross_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0))
net_rate = (gross_rate / (1 + self.vat_rate)).quantize(Decimal('0.01'))
# Build tariffs summary for all options
tariffs_data = self._build_tariffs_summary(tariffs_available, for_date)
results[category_id] = [{
'date': for_date,
'gross_rate': float(gross_rate),
'net_rate': float(net_rate),
'tariffs_data': tariffs_data
}]
return results

20
backend/services/scraper_backends/__init__.py Normal file
View file

@ -0,0 +1,20 @@
"""
Scraper backends for booking.com rate scraping.
Provides pluggable backends to allow switching between:
- playwright_local: Direct Playwright (default)
- playwright_proxy: Playwright with rotating proxies (future)
- apify_backend: Apify scraping service (future)
"""
from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
from .playwright_local import PlaywrightLocalBackend
__all__ = [
'ScraperBackend',
'ScraperResult',
'HotelData',
'RateData',
'AvailabilityStatus',
'PlaywrightLocalBackend',
]

152
backend/services/scraper_backends/base.py Normal file
View file

@ -0,0 +1,152 @@
"""
Abstract base class for booking.com scraper backends.
Defines the interface that all scraper backends must implement,
allowing easy switching between local Playwright, proxied Playwright,
or external services like Apify.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import date
from decimal import Decimal
from typing import List, Optional, Dict, Any
from enum import Enum
class AvailabilityStatus(str, Enum):
"""Availability status for a hotel rate."""
AVAILABLE = 'available' # Rate found, bookable
SOLD_OUT = 'sold_out' # Hotel shows no availability
NO_DATA = 'no_data' # Couldn't determine (scraper issue)
@dataclass
class RateData:
"""Rate data for a single hotel on a single date."""
hotel_id: Optional[str] = None # Our internal hotel_id (filled after DB lookup)
booking_com_id: str = '' # Hotel ID from booking.com
rate_date: date = None
availability_status: AvailabilityStatus = AvailabilityStatus.NO_DATA
rate_gross: Optional[Decimal] = None
currency: str = 'GBP'
room_type: Optional[str] = None
breakfast_included: Optional[bool] = None
free_cancellation: Optional[bool] = None
no_prepayment: Optional[bool] = None
rooms_left: Optional[int] = None # "Only X rooms left"
available_qty: Optional[int] = None # Future: from hotel page dropdown
@dataclass
class HotelData:
"""Hotel data discovered from search results."""
booking_com_id: str
name: str
booking_com_url: Optional[str] = None
star_rating: Optional[Decimal] = None
review_score: Optional[Decimal] = None
review_count: Optional[int] = None
@dataclass
class ScraperResult:
"""Result from a scraping operation."""
success: bool
blocked: bool = False # True if anti-scrape blocking detected
block_reason: Optional[str] = None # CAPTCHA, rate limit, etc.
hotels: List[HotelData] = field(default_factory=list)
rates: List[RateData] = field(default_factory=list)
error_message: Optional[str] = None
page_content_sample: Optional[str] = None # For debugging
class ScraperBackend(ABC):
"""
Abstract base class for scraper backends.
All backends must implement these methods to provide a consistent
interface for the main booking_scraper.py service.
"""
# Common block detection signals
BLOCK_SIGNALS = [
'captcha',
'unusual traffic',
'access denied',
'please verify',
'too many requests',
'are you a robot',
'verify you are human',
'security check',
]
@abstractmethod
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name (e.g., "Bowness-on-Windermere")
check_in: Check-in date
check_out: Check-out date (typically check_in + 1 for single night)
adults: Number of adults for search
pages: Number of search result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
pass
@abstractmethod
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape an individual hotel page for detailed rates.
Future expansion - not used in initial implementation.
Will provide available_qty from room dropdowns.
Args:
hotel_url: Full booking.com URL for the hotel
check_in: Check-in date
check_out: Check-out date
adults: Number of adults
Returns:
ScraperResult with detailed rate information
"""
pass
@abstractmethod
async def close(self):
"""Clean up any resources (browser instances, etc.)."""
pass
def detect_blocking(self, page_content: str) -> tuple[bool, Optional[str]]:
"""
Check if page content shows anti-scrape response.
Args:
page_content: HTML content of the page
Returns:
Tuple of (is_blocked, reason)
"""
content_lower = page_content.lower()
for signal in self.BLOCK_SIGNALS:
if signal in content_lower:
return True, signal
return False, None

401
backend/services/scraper_backends/playwright_local.py Normal file
View file

@ -0,0 +1,401 @@
"""
Local Playwright backend for booking.com scraping.
Uses Playwright with Chromium to scrape search results.
No proxy - direct connection. Suitable for low-volume scraping.
"""
import asyncio
import logging
import random
import re
from datetime import date
from decimal import Decimal, InvalidOperation
from typing import List, Optional
from urllib.parse import urlencode
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
from .base import (
ScraperBackend,
ScraperResult,
HotelData,
RateData,
AvailabilityStatus
)
logger = logging.getLogger(__name__)
class PlaywrightLocalBackend(ScraperBackend):
"""
Local Playwright backend using Chromium.
Features:
- Rotates user agents
- Random delays between requests
- Mimics human scroll behavior
- Uses data-testid selectors for stability
"""
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
]
def __init__(self, proxy_config: dict = None):
"""
Initialize the backend.
Args:
proxy_config: Optional proxy configuration (for future use)
"""
self.proxy_config = proxy_config
self._playwright = None
self._browser: Optional[Browser] = None
async def _ensure_browser(self) -> Browser:
"""Ensure browser is running, start if needed."""
if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--no-sandbox',
'--disable-dev-shm-usage',
]
)
return self._browser
async def _create_context(self) -> BrowserContext:
"""Create a new browser context with random user agent."""
browser = await self._ensure_browser()
context = await browser.new_context(
user_agent=random.choice(self.USER_AGENTS),
viewport={'width': 1920, 'height': 1080},
locale='en-GB',
timezone_id='Europe/London',
)
return context
def _build_search_url(
self,
location: str,
check_in: date,
check_out: date,
adults: int,
offset: int = 0
) -> str:
"""Build booking.com search URL with parameters."""
params = {
'ss': location,
'checkin': check_in.isoformat(),
'checkout': check_out.isoformat(),
'group_adults': adults,
'no_rooms': 1,
'group_children': 0,
}
if offset > 0:
params['offset'] = offset
return f"https://www.booking.com/searchresults.en-gb.html?{urlencode(params)}"
def _parse_price(self, price_text: str) -> Optional[Decimal]:
"""Parse price from text like '£150' or 'GBP 150'."""
if not price_text:
return None
# Remove currency symbols and extract number
cleaned = re.sub(r'[£$€,\s]', '', price_text)
# Find first number (including decimals)
match = re.search(r'[\d,]+(?:\.\d{2})?', cleaned)
if match:
try:
return Decimal(match.group().replace(',', ''))
except InvalidOperation:
return None
return None
def _extract_hotel_id(self, url: str) -> Optional[str]:
"""Extract hotel ID from booking.com URL."""
if not url:
return None
# URL format: /hotel/gb/hotel-name.en-gb.html or ?dest_id=123
# Try to extract from URL path
match = re.search(r'/hotel/[a-z]{2}/([^/]+)\.', url)
if match:
return match.group(1)
# Try dest_id parameter
match = re.search(r'dest_id=(-?\d+)', url)
if match:
return match.group(1)
return None
async def _human_like_scroll(self, page: Page):
"""Simulate human-like scrolling behavior."""
# Scroll down in increments
for _ in range(3):
await page.mouse.wheel(0, random.randint(300, 600))
await asyncio.sleep(random.uniform(0.3, 0.8))
async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
"""Extract hotel and rate data from search results page."""
hotels = []
rates = []
# Wait for property cards - booking.com uses data-testid
try:
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
except Exception as e:
logger.warning(f"No property cards found: {e}")
return hotels, rates
# Get all property cards
cards = await page.query_selector_all('[data-testid="property-card"]')
logger.info(f"Found {len(cards)} property cards")
for card in cards:
try:
hotel = HotelData(booking_com_id='', name='')
rate = RateData(rate_date=rate_date)
# Hotel name
name_el = await card.query_selector('[data-testid="title"]')
if name_el:
hotel.name = (await name_el.inner_text()).strip()
if not hotel.name:
continue # Skip if no name found
# Hotel URL and ID
link_el = await card.query_selector('[data-testid="title-link"]')
if link_el:
hotel.booking_com_url = await link_el.get_attribute('href')
hotel.booking_com_id = self._extract_hotel_id(hotel.booking_com_url) or ''
rate.booking_com_id = hotel.booking_com_id
# Star rating - look for star icons or rating text
stars_el = await card.query_selector('[data-testid="rating-stars"]')
if stars_el:
stars_text = await stars_el.get_attribute('aria-label') or ''
match = re.search(r'(\d+)', stars_text)
if match:
hotel.star_rating = Decimal(match.group(1))
# Review score
score_el = await card.query_selector('[data-testid="review-score"]')
if score_el:
score_text = await score_el.inner_text()
match = re.search(r'([\d.]+)', score_text)
if match:
try:
hotel.review_score = Decimal(match.group(1))
except InvalidOperation:
pass
# Check for no availability message FIRST
no_avail_el = await card.query_selector('[data-testid="availability-message"]')
if no_avail_el:
avail_text = (await no_avail_el.inner_text()).lower()
if 'no availability' in avail_text or 'sold out' in avail_text:
rate.availability_status = AvailabilityStatus.SOLD_OUT
hotels.append(hotel)
rates.append(rate)
continue
# Price
price_el = await card.query_selector('[data-testid="price-and-discounted-price"]')
if not price_el:
# Try alternative selector
price_el = await card.query_selector('[data-testid="price"]')
if price_el:
price_text = await price_el.inner_text()
rate.rate_gross = self._parse_price(price_text)
if rate.rate_gross:
rate.availability_status = AvailabilityStatus.AVAILABLE
# Room type
room_el = await card.query_selector('[data-testid="recommended-units"]')
if room_el:
rate.room_type = (await room_el.inner_text()).strip()
# Rate option badges - try multiple selectors
# Breakfast included
breakfast_el = await card.query_selector('[data-testid="breakfast-included"]')
if not breakfast_el:
# Check text content for breakfast mentions
card_text = (await card.inner_text()).lower()
rate.breakfast_included = 'breakfast included' in card_text
else:
rate.breakfast_included = True
# Free cancellation
cancel_el = await card.query_selector('[data-testid="cancellation-policy"]')
if cancel_el:
cancel_text = (await cancel_el.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in cancel_text
else:
card_text = (await card.inner_text()).lower()
rate.free_cancellation = 'free cancellation' in card_text
# No prepayment
prepay_el = await card.query_selector('[data-testid="no-prepayment"]')
if prepay_el:
rate.no_prepayment = True
else:
card_text = (await card.inner_text()).lower()
rate.no_prepayment = 'no prepayment' in card_text
# Rooms left / scarcity indicator
scarcity_el = await card.query_selector('[data-testid="availability-rate"]')
if scarcity_el:
scarcity_text = await scarcity_el.inner_text()
match = re.search(r'(\d+)\s*room', scarcity_text.lower())
if match:
rate.rooms_left = int(match.group(1))
hotels.append(hotel)
rates.append(rate)
except Exception as e:
logger.warning(f"Error extracting hotel data: {e}")
continue
return hotels, rates
async def scrape_location_search(
self,
location: str,
check_in: date,
check_out: date,
adults: int = 2,
pages: int = 2
) -> ScraperResult:
"""
Scrape booking.com location search results.
Args:
location: Location name
check_in: Check-in date
check_out: Check-out date (check_in + 1 for single night rate)
adults: Number of adults
pages: Number of result pages to scrape
Returns:
ScraperResult with hotels and rates found
"""
all_hotels = []
all_rates = []
seen_hotel_ids = set()
context = None
page = None
try:
context = await self._create_context()
page = await context.new_page()
for page_num in range(pages):
# Random delay between pages (3-7 seconds)
if page_num > 0:
delay = random.uniform(3, 7)
logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
await asyncio.sleep(delay)
# Build URL with offset for pagination (25 results per page)
url = self._build_search_url(
location, check_in, check_out, adults,
offset=page_num * 25
)
logger.info(f"Scraping page {page_num + 1}: {url}")
try:
await page.goto(url, wait_until='networkidle', timeout=30000)
except Exception as e:
logger.warning(f"Page load timeout, continuing: {e}")
# Check for blocking
content = await page.content()
is_blocked, reason = self.detect_blocking(content)
if is_blocked:
logger.warning(f"Blocking detected: {reason}")
return ScraperResult(
success=False,
blocked=True,
block_reason=reason,
hotels=all_hotels,
rates=all_rates,
page_content_sample=content[:1000]
)
# Human-like scrolling
await self._human_like_scroll(page)
# Extract data
hotels, rates = await self._extract_search_results(page, check_in)
# Deduplicate by booking_com_id
for hotel, rate in zip(hotels, rates):
if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids:
seen_hotel_ids.add(hotel.booking_com_id)
all_hotels.append(hotel)
all_rates.append(rate)
logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique")
return ScraperResult(
success=True,
blocked=False,
hotels=all_hotels,
rates=all_rates
)
except Exception as e:
logger.error(f"Scrape error: {e}")
return ScraperResult(
success=False,
blocked=False,
error_message=str(e),
hotels=all_hotels,
rates=all_rates
)
finally:
if page:
await page.close()
if context:
await context.close()
async def scrape_hotel_page(
self,
hotel_url: str,
check_in: date,
check_out: date,
adults: int = 2
) -> ScraperResult:
"""
Scrape individual hotel page for detailed rates.
Future expansion - placeholder for now.
Will extract available_qty from room dropdowns.
"""
# Not implemented in Phase 2a
logger.warning("scrape_hotel_page not yet implemented")
return ScraperResult(
success=False,
error_message="Hotel page scraping not yet implemented"
)
async def close(self):
"""Clean up browser resources."""
if self._browser:
await self._browser.close()
self._browser = None
if self._playwright:
await self._playwright.stop()
self._playwright = None

36
docker-compose.yml Normal file
View file

@ -0,0 +1,36 @@
services:
backend:
build: ./backend
security_opt:
- apparmor=unconfined
environment:
- DATABASE_URL=${DATABASE_URL}
- CENTRAL_AUTH_SECRET=${CENTRAL_AUTH_SECRET}
- APP_SLUG=rates
- SETTINGS_URL=${SETTINGS_URL:-}
- SETTINGS_SECRET=${SETTINGS_SECRET:-}
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 15s
timeout: 10s
retries: 10
start_period: 60s
restart: unless-stopped
frontend:
build:
context: ./frontend
args:
VITE_HOTEL_NAME: ${VITE_HOTEL_NAME:-Hotel}
security_opt:
- apparmor=unconfined
ports:
- "${FRONTEND_PORT:-3080}:80"
depends_on:
backend:
condition: service_healthy
restart: unless-stopped
networks:
default:
driver: bridge

15
frontend/Dockerfile Normal file
View file

@ -0,0 +1,15 @@
FROM node:22-alpine AS builder
ARG VITE_HOTEL_NAME=Hotel
ENV VITE_HOTEL_NAME=$VITE_HOTEL_NAME
WORKDIR /app
COPY package.json ./
RUN npm install
COPY . .
RUN npm run build
FROM nginx:alpine
COPY --from=builder /app/dist /usr/share/nginx/html/rates
COPY nginx.conf /etc/nginx/conf.d/default.conf
EXPOSE 80

12
frontend/index.html Normal file
View file

@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Rate Monitor</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

35
frontend/nginx.conf Normal file
View file

@ -0,0 +1,35 @@
server {
listen 80;
server_name _;
# 1. Central auth proxy
location /rates/api/auth/ {
proxy_pass http://10.10.10.101:3001/api/auth/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# 2. App backend (Python FastAPI on port 8000)
location /rates/api/ {
proxy_pass http://backend:8000/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Cookie $http_cookie;
proxy_read_timeout 300s;
proxy_connect_timeout 30s;
client_max_body_size 10M;
}
# 3. Health
location /rates/health {
proxy_pass http://backend:8000/health;
}
# 4. SPA fallback
location /rates/ {
root /usr/share/nginx/html;
try_files $uri $uri/ /rates/index.html;
}
}

30
frontend/package.json Normal file
View file

@ -0,0 +1,30 @@
{
"name": "rates-frontend",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"preview": "vite preview"
},
"dependencies": {
"@tanstack/react-query": "^5.32.0",
"axios": "^1.6.8",
"lucide-react": "^0.395.0",
"plotly.js": "^2.29.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-plotly.js": "^2.6.0",
"react-router-dom": "^6.22.0"
},
"devDependencies": {
"@types/plotly.js": "^2.12.29",
"@types/react": "^18.3.1",
"@types/react-dom": "^18.3.0",
"@types/react-plotly.js": "^2.6.4",
"@vitejs/plugin-react": "^4.2.1",
"typescript": "^5.4.5",
"vite": "^5.2.11"
}
}

28
frontend/src/App.tsx Normal file
View file

@ -0,0 +1,28 @@
import { Routes, Route, Navigate } from 'react-router-dom'
import AuthGate from './components/AuthGate'
import Layout from './components/Layout'
import Bookability from './pages/Bookability'
import MarketView from './pages/MarketView'
import DirectRates from './pages/DirectRates'
import RateAnalysis from './pages/RateAnalysis'
import Settings from './pages/Settings'
export default function App() {
return (
<AuthGate>
<Layout>
<Routes>
<Route path="/" element={<Navigate to="/bookability" replace />} />
<Route path="/bookability" element={<Bookability />} />
<Route path="/market" element={<MarketView />} />
<Route path="/direct" element={<DirectRates />} />
<Route path="/direct/:hotelId" element={<DirectRates />} />
<Route path="/analysis" element={<RateAnalysis />} />
<Route path="/settings" element={<Settings />} />
<Route path="/settings/:tab" element={<Settings />} />
<Route path="*" element={<Navigate to="/bookability" replace />} />
</Routes>
</Layout>
</AuthGate>
)
}

20
frontend/src/api.ts Normal file
View file

@ -0,0 +1,20 @@
import axios from 'axios'
const BASE = '/rates/api'
const api = axios.create({
baseURL: BASE,
withCredentials: true,
})
api.interceptors.response.use(
(response) => response,
(error) => {
if (error.response?.status === 401) {
window.location.href = '/auth/login?redirect=' + encodeURIComponent(window.location.pathname)
}
return Promise.reject(error)
}
)
export default api

47
frontend/src/components/AuthGate.tsx Normal file
View file

@ -0,0 +1,47 @@
import { createContext, useContext, useEffect, useState } from 'react'
import type { ReactNode } from 'react'
import type { User } from '../types'
interface AuthCtx { user: User }
const Ctx = createContext<AuthCtx | null>(null)
export function useAuth() {
const ctx = useContext(Ctx)
if (!ctx) throw new Error('useAuth outside AuthGate')
return ctx
}
export default function AuthGate({ children }: { children: ReactNode }) {
const [user, setUser] = useState<User | null>(null)
const [checking, setChecking] = useState(true)
useEffect(() => {
fetch('/rates/api/auth/verify?app=rates', { credentials: 'include' })
.then(r => {
if (!r.ok) throw new Error('unauth')
return r.json()
})
.then(data => setUser({
email: data.email || data.sub || '',
name: data.name || data.display_name || '',
is_admin: data.is_admin ?? false,
caps: data.caps ?? [],
}))
.catch(() => {
window.location.href = '/auth/login?redirect=' + encodeURIComponent(window.location.pathname)
})
.finally(() => setChecking(false))
}, [])
if (checking) {
return (
<div style={{ minHeight: '100vh', display: 'flex', alignItems: 'center', justifyContent: 'center', background: 'var(--navy-dark)' }}>
<div className="spinner" style={{ width: 32, height: 32 }} />
</div>
)
}
if (!user) return null
return <Ctx.Provider value={{ user }}>{children}</Ctx.Provider>
}

70
frontend/src/components/Layout.tsx Normal file
View file

@ -0,0 +1,70 @@
import { NavLink } from 'react-router-dom'
import { TrendingUp, Calendar, Globe, BarChart2, Settings, Building2 } from 'lucide-react'
import { useQuery } from '@tanstack/react-query'
import { useAuth } from './AuthGate'
import { can } from '../types'
import api from '../api'
import type { ReactNode } from 'react'
const ICON = { size: 16, strokeWidth: 1.75 }
const NAV = [
{ to: '/bookability', label: 'Bookability', icon: Calendar, cap: 'view_own_rates' },
{ to: '/market', label: 'Market View', icon: Globe, cap: 'view_competitors' },
{ to: '/direct', label: 'Direct Rates', icon: Building2, cap: 'view_direct_rates' },
{ to: '/analysis', label: 'Rate Analysis', icon: TrendingUp, cap: 'rate_analysis' },
{ to: '/settings', label: 'Settings', icon: Settings, cap: 'manage_scraper' },
]
export default function Layout({ children }: { children: ReactNode }) {
const { user } = useAuth()
const items = NAV.filter(n => can(user, n.cap))
const { data: alertCount } = useQuery<number>({
queryKey: ['parity-alert-count'],
queryFn: () => api.get('/competitors/parity/alerts?status=active').then(r => r.data.length),
refetchInterval: 60_000,
enabled: can(user, 'view_competitors'),
})
return (
<div className="app-shell">
<aside className="sidebar">
<div className="sidebar-logo">
<BarChart2 size={18} strokeWidth={1.75} />
Rate Monitor
</div>
<nav className="sidebar-nav">
{items.map(({ to, label, icon: Icon }) => (
<NavLink key={to} to={to} className={({ isActive }) => isActive ? 'active' : ''}>
<Icon {...ICON} />
{label}
{to === '/market' && alertCount ? (
<span style={{
marginLeft: 'auto', background: 'var(--danger)', color: '#fff',
borderRadius: 10, fontSize: 10, fontWeight: 700,
padding: '1px 6px', lineHeight: '16px',
}}>{alertCount}</span>
) : null}
</NavLink>
))}
</nav>
<div className="sidebar-user">{user.name || user.email}</div>
</aside>
<header className="top-bar">
<BarChart2 size={18} strokeWidth={1.75} color="var(--gold)" />
<span className="top-bar-title">Rate Monitor</span>
<nav className="top-bar-nav">
{items.map(({ to, label }) => (
<NavLink key={to} to={to} className={({ isActive }) => isActive ? 'active' : ''}>
{label}
</NavLink>
))}
</nav>
</header>
<main className="page-content">{children}</main>
</div>
)
}

363
frontend/src/index.css Normal file
View file

@ -0,0 +1,363 @@
:root {
--navy: #1a1a2e;
--navy-dark: #0f0f20;
--gold: #c9a84c;
--gold-light: #e8c96d;
--surface: rgba(255,255,255,0.07);
--surface-2: rgba(255,255,255,0.08);
--text: rgba(255,255,255,0.88);
--text-muted: rgba(255,255,255,0.48);
--body-bg: #f4f5f7;
--card-bg: #ffffff;
--card-border: #e4e8ee;
--text-dark: #1e293b;
--text-mid: #64748b;
--shadow-sm: 0 1px 3px rgba(0,0,0,0.07), 0 1px 2px rgba(0,0,0,0.04);
--shadow-md: 0 4px 12px rgba(0,0,0,0.08);
--danger: #dc2626;
--success: #16a34a;
--warning: #d97706;
--radius: 10px;
--font: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
}
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
body {
background: var(--body-bg);
color: var(--text-dark);
font-family: var(--font);
font-size: 14px;
line-height: 1.5;
}
/* App shell layout */
.app-shell {
display: grid;
grid-template-columns: 220px 1fr;
grid-template-rows: auto 1fr;
min-height: 100vh;
}
.sidebar {
grid-column: 1;
grid-row: 1 / -1;
background: var(--navy);
display: flex;
flex-direction: column;
padding: 0;
position: sticky;
top: 0;
height: 100vh;
overflow-y: auto;
}
.sidebar-logo {
display: flex;
align-items: center;
gap: 10px;
padding: 20px 16px 16px;
font-size: 15px;
font-weight: 600;
color: var(--text);
border-bottom: 1px solid var(--surface);
}
.sidebar-nav {
flex: 1;
padding: 12px 8px;
display: flex;
flex-direction: column;
gap: 2px;
}
.sidebar-nav a {
display: flex;
align-items: center;
gap: 10px;
padding: 9px 10px;
border-radius: 8px;
color: var(--text-muted);
text-decoration: none;
font-size: 13.5px;
transition: background 0.15s, color 0.15s;
}
.sidebar-nav a:hover { background: var(--surface); color: var(--text); }
.sidebar-nav a.active { background: rgba(201,168,76,0.15); color: var(--gold); }
.sidebar-user {
padding: 12px 16px;
font-size: 12px;
color: var(--text-muted);
border-top: 1px solid var(--surface);
}
/* Top bar — mobile/collapsed fallback */
.top-bar {
display: none;
}
.page-content {
grid-column: 2;
grid-row: 1 / -1;
min-width: 0;
padding: 24px;
background: var(--body-bg);
}
/* Cards */
.card {
background: var(--card-bg);
border: 1px solid var(--card-border);
border-radius: var(--radius);
box-shadow: var(--shadow-sm);
}
.card-header {
padding: 16px 20px;
border-bottom: 1px solid var(--card-border);
font-size: 14px;
font-weight: 600;
color: var(--text-dark);
display: flex;
align-items: center;
justify-content: space-between;
}
.card-body { padding: 20px; }
/* Buttons */
.btn {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 7px 14px;
border-radius: 7px;
font-size: 13px;
font-weight: 500;
cursor: pointer;
border: none;
transition: background 0.15s, opacity 0.15s;
}
.btn:disabled { opacity: 0.5; cursor: not-allowed; }
.btn-primary {
background: var(--gold);
color: var(--navy);
}
.btn-primary:hover:not(:disabled) { background: var(--gold-light); }
.btn-secondary {
background: var(--navy);
color: var(--text);
}
.btn-secondary:hover:not(:disabled) { background: var(--navy-dark); }
.btn-outline {
background: transparent;
color: var(--text-dark);
border: 1px solid var(--card-border);
}
.btn-outline:hover:not(:disabled) { background: var(--body-bg); }
.btn-danger {
background: var(--danger);
color: white;
}
.btn-danger:hover:not(:disabled) { opacity: 0.85; }
.btn-sm { padding: 4px 10px; font-size: 12px; }
/* Form elements */
input, select, textarea {
background: var(--card-bg);
border: 1px solid var(--card-border);
border-radius: 7px;
padding: 7px 10px;
font-size: 13px;
color: var(--text-dark);
width: 100%;
outline: none;
transition: border-color 0.15s;
}
input:focus, select:focus, textarea:focus { border-color: var(--gold); }
/* Badges */
.badge {
display: inline-flex;
align-items: center;
padding: 2px 8px;
border-radius: 20px;
font-size: 11px;
font-weight: 500;
}
.badge-success { background: #dcfce7; color: #16a34a; }
.badge-warning { background: #fef3c7; color: #d97706; }
.badge-danger { background: #fee2e2; color: #dc2626; }
.badge-info { background: #dbeafe; color: #2563eb; }
.badge-neutral { background: #f1f5f9; color: #64748b; }
/* Tables */
.table-wrap { overflow-x: auto; }
table { width: 100%; border-collapse: collapse; font-size: 13px; }
th {
text-align: left;
padding: 8px 12px;
background: #f8fafc;
border-bottom: 1px solid var(--card-border);
font-weight: 600;
color: var(--text-mid);
white-space: nowrap;
}
td {
padding: 8px 12px;
border-bottom: 1px solid #f1f5f9;
color: var(--text-dark);
}
tr:last-child td { border-bottom: none; }
tr:hover td { background: #f8fafc; }
/* Page header */
.page-header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 24px;
gap: 16px;
}
.page-title {
font-size: 20px;
font-weight: 700;
color: var(--text-dark);
}
.page-subtitle { font-size: 13px; color: var(--text-mid); margin-top: 2px; }
/* Sub-nav tabs */
.sub-nav {
display: flex;
gap: 4px;
border-bottom: 1px solid var(--card-border);
margin-bottom: 24px;
overflow-x: auto;
}
.sub-nav-item {
padding: 8px 14px;
font-size: 13px;
font-weight: 500;
color: var(--text-mid);
cursor: pointer;
border-bottom: 2px solid transparent;
white-space: nowrap;
background: none;
border-left: none;
border-right: none;
border-top: none;
transition: color 0.15s, border-color 0.15s;
}
.sub-nav-item:hover { color: var(--text-dark); }
.sub-nav-item.active { color: var(--gold); border-bottom-color: var(--gold); }
/* Status dot */
.status-dot {
width: 8px; height: 8px;
border-radius: 50%;
display: inline-block;
}
.status-dot.green { background: var(--success); }
.status-dot.yellow { background: var(--warning); }
.status-dot.red { background: var(--danger); }
.status-dot.grey { background: #94a3b8; }
/* Spinner */
.spinner {
width: 20px; height: 20px;
border: 2px solid var(--card-border);
border-top-color: var(--gold);
border-radius: 50%;
animation: spin 0.7s linear infinite;
display: inline-block;
}
@keyframes spin { to { transform: rotate(360deg); } }
/* Loading / empty states */
.loading-state {
display: flex;
align-items: center;
justify-content: center;
gap: 10px;
padding: 60px 20px;
color: var(--text-mid);
font-size: 13px;
}
.empty-state {
text-align: center;
padding: 60px 20px;
color: var(--text-mid);
font-size: 13px;
}
/* Grid helpers */
.grid-2 { display: grid; grid-template-columns: repeat(2, 1fr); gap: 16px; }
.grid-3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }
.grid-4 { display: grid; grid-template-columns: repeat(4, 1fr); gap: 16px; }
/* Stat card */
.stat-card {
background: var(--card-bg);
border: 1px solid var(--card-border);
border-radius: var(--radius);
padding: 16px 20px;
box-shadow: var(--shadow-sm);
}
.stat-label { font-size: 12px; color: var(--text-mid); margin-bottom: 4px; }
.stat-value { font-size: 24px; font-weight: 700; color: var(--text-dark); }
.stat-delta { font-size: 12px; margin-top: 4px; }
.stat-delta.positive { color: var(--success); }
.stat-delta.negative { color: var(--danger); }
.stat-delta.neutral { color: var(--text-mid); }
/* Responsive — at narrow widths hide sidebar, show top-bar */
@media (max-width: 900px) {
.app-shell {
grid-template-columns: 1fr;
grid-template-rows: auto 1fr;
}
.sidebar { display: none; }
.top-bar {
display: flex;
align-items: center;
gap: 12px;
background: var(--navy);
padding: 0 16px;
height: 52px;
grid-column: 1;
grid-row: 1;
overflow-x: auto;
}
.top-bar-title {
font-size: 14px;
font-weight: 600;
color: var(--text);
white-space: nowrap;
margin-right: 8px;
}
.top-bar-nav { display: flex; gap: 4px; }
.top-bar-nav a {
color: var(--text-muted);
text-decoration: none;
font-size: 12.5px;
padding: 6px 10px;
border-radius: 6px;
white-space: nowrap;
}
.top-bar-nav a:hover { color: var(--text); background: var(--surface); }
.top-bar-nav a.active { color: var(--gold); }
.page-content {
grid-column: 1;
grid-row: 2;
padding: 16px;
}
}

20
frontend/src/main.tsx Normal file
View file

@ -0,0 +1,20 @@
import React from 'react'
import ReactDOM from 'react-dom/client'
import { BrowserRouter } from 'react-router-dom'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import App from './App'
import './index.css'
const qc = new QueryClient({
defaultOptions: { queries: { retry: 1, staleTime: 30_000 } },
})
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<BrowserRouter basename="/rates">
<QueryClientProvider client={qc}>
<App />
</QueryClientProvider>
</BrowserRouter>
</React.StrictMode>,
)

1094
frontend/src/pages/Bookability.tsx Normal file
View file

File diff suppressed because it is too large Load diff

520
frontend/src/pages/DirectRates.tsx Normal file
View file

@ -0,0 +1,520 @@
import React, { useState } from 'react'
import { useParams, useNavigate } from 'react-router-dom'
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
import {
Building2, RefreshCw, Plus, Settings, ChevronDown, ChevronRight,
AlertCircle, CheckCircle, Clock,
} from 'lucide-react'
import api from '../api'
import { useAuth } from '../components/AuthGate'
import { can } from '../types'
const fmtDate = (d: Date) =>
`${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
const TABS = ['overview', 'hotel', 'manage'] as const
type Tab = typeof TABS[number]
interface DirectHotel {
id: number
name: string
profile_name: string
scrape_enabled: boolean
last_scraped_at: string | null
scraped_dates: number
last_rate_at: string | null
}
interface DateRow {
stay_date: string
cheapest_rate: number | null
has_availability: boolean
has_min_stay: boolean
scraped_at: string | null
}
interface RoomRow {
room_id: string
rate_id: string
room_label: string
rate_label: string
availability: number
price_incl: number | null
min_stay_nights: number | null
bench_rate: number | null
}
const fmt = (v: number | null) => v != null ? `£${v.toFixed(2)}` : '—'
const age = (ts: string | null) => {
if (!ts) return 'Never'
const h = Math.round((Date.now() - new Date(ts).getTime()) / 3600000)
return h < 24 ? `${h}h ago` : `${Math.round(h / 24)}d ago`
}
export default function DirectRates() {
const { hotelId } = useParams<{ hotelId?: string }>()
const navigate = useNavigate()
const { user } = useAuth()
const canManage = can(user, 'manage_hotels')
const [tab, setTab] = useState<Tab>(hotelId ? 'hotel' : 'overview')
const [selectedHotel, setSelectedHotel] = useState<number | null>(hotelId ? parseInt(hotelId) : null)
const [expandedDate, setExpandedDate] = useState<string | null>(null)
const [fromDate, setFromDate] = useState(fmtDate(new Date()))
const [toDate, setToDate] = useState(fmtDate(new Date(Date.now() + 89 * 86400000)))
const { data: hotels, isLoading: hotelsLoading } = useQuery<DirectHotel[]>({
queryKey: ['direct-hotels'],
queryFn: () => api.get('/direct/hotels').then(r => r.data),
})
const { data: dates, isLoading: datesLoading } = useQuery<{ dates: DateRow[]; hotel_name: string }>({
queryKey: ['direct-dates', selectedHotel, fromDate, toDate],
queryFn: () => api.get(`/direct/hotels/${selectedHotel}/dates`, {
params: { from_date: fromDate, to_date: toDate }
}).then(r => r.data),
enabled: !!selectedHotel && tab === 'hotel',
})
const { data: roomData } = useQuery<{ rooms: RoomRow[]; bench_price: number | null }>({
queryKey: ['direct-rooms', selectedHotel, expandedDate],
queryFn: () => api.get(`/direct/hotels/${selectedHotel}/date/${expandedDate}/rooms`).then(r => r.data),
enabled: !!selectedHotel && !!expandedDate,
})
const qc = useQueryClient()
const scrapeMutation = useMutation({
mutationFn: (id: number) => api.post(`/direct/hotels/${id}/scrape`),
onSuccess: () => qc.invalidateQueries({ queryKey: ['direct-hotels'] }),
})
const selectHotel = (id: number) => {
setSelectedHotel(id)
setTab('hotel')
setExpandedDate(null)
navigate(`/direct/${id}`)
}
const presets = [
{ label: '7d', days: 7 }, { label: '14d', days: 14 },
{ label: '30d', days: 30 }, { label: '90d', days: 90 },
]
return (
<div>
<div className="page-header">
<div>
<div className="page-title">Direct Rates</div>
<div className="page-subtitle">Competitor booking engine rates — scraped directly</div>
</div>
</div>
<div className="sub-nav">
{TABS.map(t => (
<button key={t} className={`sub-nav-item${tab === t ? ' active' : ''}`}
onClick={() => setTab(t)}>
{t === 'overview' ? 'Overview' : t === 'hotel' ? 'Hotel Detail' : 'Manage'}
</button>
))}
</div>
{tab === 'overview' && (
<OverviewTab
hotels={hotels || []}
loading={hotelsLoading}
onSelectHotel={selectHotel}
onScrape={id => scrapeMutation.mutate(id)}
scraping={scrapeMutation.isPending}
/>
)}
{tab === 'hotel' && (
<HotelDetailTab
hotels={hotels || []}
selectedHotel={selectedHotel}
onSelectHotel={selectHotel}
dates={dates?.dates || []}
hotelName={dates?.hotel_name}
datesLoading={datesLoading}
fromDate={fromDate}
toDate={toDate}
setFromDate={setFromDate}
setToDate={setToDate}
presets={presets}
expandedDate={expandedDate}
setExpandedDate={setExpandedDate}
roomData={roomData}
/>
)}
{tab === 'manage' && canManage && (
<ManageTab hotels={hotels || []} onRefresh={() => qc.invalidateQueries({ queryKey: ['direct-hotels'] })} />
)}
{tab === 'manage' && !canManage && (
<div className="empty-state">You don't have permission to manage competitor hotels.</div>
)}
</div>
)
}
// ─── Overview Tab ─────────────────────────────────────────────────────────────
function OverviewTab({ hotels, loading, onSelectHotel, onScrape, scraping }: {
hotels: DirectHotel[]
loading: boolean
onSelectHotel: (id: number) => void
onScrape: (id: number) => void
scraping: boolean
}) {
if (loading) return <div className="loading-state"><div className="spinner" />Loading…</div>
if (!hotels.length) return (
<div className="empty-state">
<Building2 size={32} strokeWidth={1.5} style={{ margin: '0 auto 12px', display: 'block', color: 'var(--text-mid)' }} />
No competitor hotels configured. Use the Manage tab to add hotels.
</div>
)
return (
<div className="card">
<div className="table-wrap">
<table>
<thead>
<tr>
<th>Hotel</th>
<th>Engine</th>
<th>Dates Scraped</th>
<th>Last Scrape</th>
<th>Status</th>
<th></th>
</tr>
</thead>
<tbody>
{hotels.map(h => (
<tr key={h.id}>
<td>
<button
onClick={() => onSelectHotel(h.id)}
style={{ background: 'none', border: 'none', cursor: 'pointer', color: 'var(--gold)', fontWeight: 600, padding: 0, fontSize: 13 }}
>
{h.name}
</button>
</td>
<td><span className="badge badge-neutral">{h.profile_name}</span></td>
<td>{h.scraped_dates}</td>
<td style={{ color: 'var(--text-mid)', fontSize: 12 }}>{age(h.last_scraped_at)}</td>
<td>
<span className={`badge ${h.scrape_enabled ? 'badge-success' : 'badge-neutral'}`}>
{h.scrape_enabled ? 'Active' : 'Paused'}
</span>
</td>
<td>
<button className="btn btn-outline btn-sm" onClick={() => onScrape(h.id)} disabled={scraping}>
<RefreshCw size={12} strokeWidth={1.75} />
Scrape
</button>
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)
}
// ─── Hotel Detail Tab ─────────────────────────────────────────────────────────
function HotelDetailTab({ hotels, selectedHotel, onSelectHotel, dates, hotelName, datesLoading,
fromDate, toDate, setFromDate, setToDate, presets, expandedDate, setExpandedDate, roomData }: any) {
return (
<div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
{/* Controls */}
<div style={{ display: 'flex', gap: 12, flexWrap: 'wrap', alignItems: 'flex-end' }}>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>Hotel</label>
<select
style={{ width: 220 }}
value={selectedHotel || ''}
onChange={e => onSelectHotel(parseInt(e.target.value))}
>
<option value="">Select a hotel…</option>
{hotels.map((h: DirectHotel) => (
<option key={h.id} value={h.id}>{h.name}</option>
))}
</select>
</div>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>From</label>
<input type="date" style={{ width: 140 }} value={fromDate} onChange={e => setFromDate(e.target.value)} />
</div>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>To</label>
<input type="date" style={{ width: 140 }} value={toDate} onChange={e => setToDate(e.target.value)} />
</div>
<div style={{ display: 'flex', gap: 6 }}>
{presets.map((p: any) => (
<button key={p.label} className="btn btn-outline btn-sm"
onClick={() => {
const from = new Date(); const to = new Date(Date.now() + p.days * 86400000)
setFromDate(fmtDate(from)); setToDate(fmtDate(to))
}}>{p.label}</button>
))}
</div>
</div>
{!selectedHotel && <div className="empty-state">Select a hotel to view rates.</div>}
{selectedHotel && datesLoading && <div className="loading-state"><div className="spinner" />Loading…</div>}
{selectedHotel && !datesLoading && dates.length === 0 && (
<div className="empty-state">No rate data for this period. Run a scrape first.</div>
)}
{selectedHotel && !datesLoading && dates.length > 0 && (
<div className="card">
<div className="card-header">{hotelName} — {dates.length} dates</div>
<div className="table-wrap">
<table>
<thead>
<tr>
<th></th>
<th>Date</th>
<th>Cheapest Rate</th>
<th>Availability</th>
<th>Min-Stay</th>
<th>Scraped</th>
</tr>
</thead>
<tbody>
{dates.map((d: DateRow) => (
<React.Fragment key={d.stay_date}>
<tr
onClick={() => setExpandedDate(expandedDate === d.stay_date ? null : d.stay_date)}
style={{ cursor: 'pointer' }}
>
<td style={{ width: 24, color: 'var(--text-mid)' }}>
{expandedDate === d.stay_date
? <ChevronDown size={14} strokeWidth={1.75} />
: <ChevronRight size={14} strokeWidth={1.75} />}
</td>
<td>{d.stay_date}</td>
<td style={{ fontWeight: 600 }}>
{d.cheapest_rate ? `£${Number(d.cheapest_rate).toFixed(2)}` : '—'}
</td>
<td>
{d.has_availability
? <CheckCircle size={14} strokeWidth={1.75} color="var(--success)" />
: <AlertCircle size={14} strokeWidth={1.75} color="var(--danger)" />}
</td>
<td>
{d.has_min_stay
? <span className="badge badge-warning">Min-stay</span>
: null}
</td>
<td style={{ fontSize: 11, color: 'var(--text-mid)' }}>{age(d.scraped_at)}</td>
</tr>
{expandedDate === d.stay_date && roomData && (
<tr>
<td colSpan={6} style={{ background: '#f8fafc', padding: '8px 16px' }}>
<RoomBreakdown rooms={roomData.rooms} benchPrice={roomData.bench_price} />
</td>
</tr>
)}
</React.Fragment>
))}
</tbody>
</table>
</div>
</div>
)}
</div>
)
}
function RoomBreakdown({ rooms, benchPrice }: { rooms: RoomRow[]; benchPrice: number | null }) {
return (
<table style={{ width: '100%', fontSize: 12 }}>
<thead>
<tr>
<th>Room</th>
<th>Rate Plan</th>
<th>Avail</th>
<th>Price</th>
<th>Bench Rate</th>
<th>Min-Stay</th>
</tr>
</thead>
<tbody>
{rooms.map((r, i) => (
<tr key={i}>
<td>{r.room_label}</td>
<td style={{ color: 'var(--text-mid)' }}>{r.rate_label}</td>
<td>{r.availability > 0 ? <CheckCircle size={12} strokeWidth={1.75} color="var(--success)" /> : <AlertCircle size={12} strokeWidth={1.75} color="var(--danger)" />}</td>
<td style={{ fontWeight: 600 }}>{fmt(r.price_incl)}</td>
<td style={{ color: 'var(--text-mid)' }}>{fmt(r.bench_rate)}</td>
<td>{r.min_stay_nights && r.min_stay_nights > 1 ? <span className="badge badge-warning">{r.min_stay_nights}N</span> : null}</td>
</tr>
))}
</tbody>
</table>
)
}
// ─── Manage Tab ───────────────────────────────────────────────────────────────
function ManageTab({ hotels, onRefresh }: { hotels: DirectHotel[]; onRefresh: () => void }) {
const [showAdd, setShowAdd] = useState(false)
const [detectUrl, setDetectUrl] = useState('')
const [detected, setDetected] = useState<any>(null)
const [detecting, setDetecting] = useState(false)
const [newName, setNewName] = useState('')
const [extraParams, setExtraParams] = useState<Record<string, string>>({})
const [profiles, setProfiles] = useState<any[]>([])
const qc = useQueryClient()
const { data: profileList } = useQuery({
queryKey: ['direct-profiles'],
queryFn: () => api.get('/direct/profiles').then(r => r.data),
})
const detectMutation = useMutation({
mutationFn: (url: string) => api.post('/direct/profiles/detect', { url }),
onSuccess: (res) => setDetected(res.data),
})
const createMutation = useMutation({
mutationFn: (body: any) => api.post('/direct/hotels', body),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['direct-hotels'] }); setShowAdd(false); setDetected(null); setDetectUrl('') },
})
const toggleMutation = useMutation({
mutationFn: ({ id, enabled }: { id: number; enabled: boolean }) =>
api.put(`/direct/hotels/${id}`, { scrape_enabled: enabled }),
onSuccess: () => qc.invalidateQueries({ queryKey: ['direct-hotels'] }),
})
const discoveryMutation = useMutation({
mutationFn: (id: number) => api.post(`/direct/hotels/${id}/discover`),
})
const requiredParams = detected
? (profileList || []).find((p: any) => p.name === detected.profile)?.required_params || []
: []
return (
<div style={{ display: 'flex', flexDirection: 'column', gap: 20 }}>
<div style={{ display: 'flex', justifyContent: 'flex-end' }}>
<button className="btn btn-primary" onClick={() => setShowAdd(!showAdd)}>
<Plus size={14} strokeWidth={1.75} />
Add Competitor
</button>
</div>
{showAdd && (
<div className="card">
<div className="card-header">Add Competitor Hotel</div>
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 14 }}>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 6 }}>
Booking URL (paste any booking page URL to auto-detect engine)
</label>
<div style={{ display: 'flex', gap: 8 }}>
<input value={detectUrl} onChange={e => setDetectUrl(e.target.value)}
placeholder="https://booking.eu.guestline.app/..." />
<button className="btn btn-outline"
onClick={() => detectMutation.mutate(detectUrl)}
disabled={!detectUrl || detectMutation.isPending}>
Detect
</button>
</div>
</div>
{detected && (
<>
<div style={{ padding: '10px 14px', background: '#f0fdf4', borderRadius: 8, border: '1px solid #bbf7d0', fontSize: 13 }}>
<strong>Detected:</strong> {detected.profile} engine
{Object.entries(detected).filter(([k]) => k !== 'profile').map(([k, v]) => (
<span key={k} style={{ marginLeft: 12, color: 'var(--text-mid)' }}>{k}: <strong>{String(v)}</strong></span>
))}
</div>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 6 }}>Hotel Name</label>
<input value={newName} onChange={e => setNewName(e.target.value)} placeholder="e.g. Three Ways House Hotel" style={{ maxWidth: 300 }} />
</div>
{requiredParams.filter((p: any) => !(p.key in detected)).map((p: any) => (
<div key={p.key}>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>{p.label}</label>
<input
placeholder={p.help}
style={{ maxWidth: 300 }}
value={extraParams[p.key] || ''}
onChange={e => setExtraParams(prev => ({ ...prev, [p.key]: e.target.value }))}
/>
</div>
))}
<button
className="btn btn-primary"
onClick={() => createMutation.mutate({
name: newName,
profile_name: detected.profile,
params: { ...detected, ...extraParams, profile: undefined },
})}
disabled={!newName || createMutation.isPending}
>
Add Hotel
</button>
</>
)}
</div>
</div>
)}
<div className="card">
<div className="card-header">Configured Competitors</div>
<div className="table-wrap">
<table>
<thead>
<tr>
<th>Name</th>
<th>Engine</th>
<th>Scraping</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{hotels.length === 0 && (
<tr><td colSpan={4} style={{ textAlign: 'center', color: 'var(--text-mid)', padding: 24 }}>No competitors configured yet.</td></tr>
)}
{hotels.map(h => (
<tr key={h.id}>
<td style={{ fontWeight: 500 }}>{h.name}</td>
<td><span className="badge badge-neutral">{h.profile_name}</span></td>
<td>
<button
className={`btn btn-sm ${h.scrape_enabled ? 'btn-outline' : 'btn-primary'}`}
onClick={() => toggleMutation.mutate({ id: h.id, enabled: !h.scrape_enabled })}
>
{h.scrape_enabled ? 'Enabled' : 'Disabled'}
</button>
</td>
<td style={{ display: 'flex', gap: 6 }}>
<button className="btn btn-outline btn-sm"
onClick={() => discoveryMutation.mutate(h.id)}
disabled={discoveryMutation.isPending}>
<Clock size={12} strokeWidth={1.75} />
Re-discover
</button>
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</div>
)
}

1940
frontend/src/pages/MarketView.tsx Normal file
View file

File diff suppressed because it is too large Load diff

362
frontend/src/pages/RateAnalysis.tsx Normal file
View file

@ -0,0 +1,362 @@
import { useState } from 'react'
import { useQuery } from '@tanstack/react-query'
import Plot from 'react-plotly.js'
import { TrendingUp, TrendingDown, Minus, AlertTriangle, ChevronDown } from 'lucide-react'
import api from '../api'
const fmtDate = (d: Date) =>
`${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
interface AnalysisHotel {
hotel_id: number
hotel_name: string
last_scraped: string | null
date_count: number
}
interface StrategyLabel {
label: string
advance_discount_pct: number
weekend_premium_pct: number
avg_sold_out_rate_pct: number
peak_months: string[]
}
interface HotelAnalysis {
strategy: StrategyLabel
advance_curve: { days_ahead: number; avg_price: number; sample_count: number }[]
dow_breakdown: { dow: number; dow_name: string; avg_price: number; count: number }[]
sold_out_pattern: { stay_date: string; sold_out_pct: number }[]
}
interface TimelineEntry {
scraped_at: string
room_id: string
rate_id: string
room_label: string
rate_label: string
price_incl: number | null
availability: number
}
interface ComparisonRow {
hotel_id: number
hotel_name: string
our_rate: number | null
their_rate: number | null
price_index: number | null
days_checked: number
}
const DOW = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
const PLOT_LAYOUT_BASE = {
paper_bgcolor: 'transparent',
plot_bgcolor: 'transparent',
font: { family: 'Inter, system-ui, sans-serif', size: 12, color: '#60748b' },
margin: { t: 20, r: 16, b: 48, l: 48 },
showlegend: false,
xaxis: { gridcolor: '#e5e9f0', zeroline: false },
yaxis: { gridcolor: '#e5e9f0', zeroline: false },
}
function strategyIcon(label: string) {
if (label.includes('Discount')) return <TrendingDown size={16} strokeWidth={1.75} color="var(--warning)" />
if (label.includes('Premium')) return <TrendingUp size={16} strokeWidth={1.75} color="var(--success)" />
if (label.includes('Yield')) return <TrendingUp size={16} strokeWidth={1.75} color="var(--gold)" />
return <Minus size={16} strokeWidth={1.75} color="var(--text-mid)" />
}
function priceIndexClass(idx: number | null) {
if (idx == null) return 'badge badge-neutral'
if (idx > 105) return 'badge badge-success'
if (idx < 85) return 'badge badge-danger'
if (idx < 95) return 'badge badge-warning'
return 'badge badge-neutral'
}
export default function RateAnalysis() {
const [selectedHotel, setSelectedHotel] = useState<number | null>(null)
const [timelineDate, setTimelineDate] = useState(fmtDate(new Date(Date.now() + 30 * 86400000)))
const [compFrom, setCompFrom] = useState(fmtDate(new Date()))
const [compTo, setCompTo] = useState(fmtDate(new Date(Date.now() + 29 * 86400000)))
const { data: hotels } = useQuery<AnalysisHotel[]>({
queryKey: ['analysis-hotels'],
queryFn: () => api.get('/analysis/hotels').then(r => r.data),
})
const { data: analysis, isLoading: analysisLoading } = useQuery<HotelAnalysis>({
queryKey: ['analysis-hotel', selectedHotel],
queryFn: () => api.get(`/analysis/hotel/${selectedHotel}`).then(r => r.data),
enabled: !!selectedHotel,
})
const { data: timeline } = useQuery<TimelineEntry[]>({
queryKey: ['analysis-timeline', selectedHotel, timelineDate],
queryFn: () => api.get(`/analysis/hotel/${selectedHotel}/timeline`, {
params: { date: timelineDate }
}).then(r => r.data),
enabled: !!selectedHotel,
})
const { data: comparison, isLoading: compLoading } = useQuery<ComparisonRow[]>({
queryKey: ['analysis-comparison', compFrom, compTo],
queryFn: () => api.get('/analysis/comparison', {
params: { from_date: compFrom, to_date: compTo }
}).then(r => r.data),
enabled: !!(compFrom && compTo),
})
const presets = [
{ label: '7d', days: 7 }, { label: '14d', days: 14 },
{ label: '30d', days: 30 },
]
return (
<div>
<div className="page-header">
<div>
<div className="page-title">Rate Analysis</div>
<div className="page-subtitle">Competitor pricing structure and advance purchase behaviour</div>
</div>
</div>
{/* Comparison table — full width, no hotel needed */}
<section style={{ marginBottom: 24 }}>
<div className="card">
<div className="card-header" style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<span>Market Comparison</span>
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
{presets.map(p => (
<button key={p.label} className="btn btn-outline btn-sm"
onClick={() => { setCompFrom(fmtDate(new Date())); setCompTo(fmtDate(new Date(Date.now() + p.days * 86400000))) }}>
{p.label}
</button>
))}
<input type="date" style={{ width: 130 }} value={compFrom} onChange={e => setCompFrom(e.target.value)} />
<span style={{ color: 'var(--text-mid)', fontSize: 12 }}>to</span>
<input type="date" style={{ width: 130 }} value={compTo} onChange={e => setCompTo(e.target.value)} />
</div>
</div>
{compLoading ? (
<div className="loading-state"><div className="spinner" />Loading…</div>
) : (
<div className="table-wrap">
<table>
<thead>
<tr>
<th>Competitor</th>
<th>Our Avg Rate</th>
<th>Their Avg Rate</th>
<th>Price Index</th>
<th>Dates Checked</th>
</tr>
</thead>
<tbody>
{(comparison || []).length === 0 && (
<tr><td colSpan={5} style={{ textAlign: 'center', color: 'var(--text-mid)', padding: 24 }}>No comparison data available.</td></tr>
)}
{(comparison || []).map(row => (
<tr key={row.hotel_id}>
<td>
<button onClick={() => setSelectedHotel(row.hotel_id)}
style={{ background: 'none', border: 'none', cursor: 'pointer', color: 'var(--gold)', fontWeight: 600, padding: 0, fontSize: 13 }}>
{row.hotel_name}
</button>
</td>
<td>{row.our_rate ? `£${Number(row.our_rate).toFixed(2)}` : '—'}</td>
<td style={{ fontWeight: 600 }}>{row.their_rate ? `£${Number(row.their_rate).toFixed(2)}` : '—'}</td>
<td>
{row.price_index != null ? (
<span className={priceIndexClass(row.price_index)}>
{row.price_index.toFixed(0)}
</span>
) : '—'}
</td>
<td style={{ color: 'var(--text-mid)' }}>{row.days_checked}</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</div>
</section>
{/* Hotel selector for deep analysis */}
<div style={{ marginBottom: 20 }}>
<div style={{ display: 'flex', gap: 12, alignItems: 'flex-end' }}>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
Competitor — deep analysis
</label>
<select style={{ width: 260 }} value={selectedHotel || ''}
onChange={e => setSelectedHotel(e.target.value ? parseInt(e.target.value) : null)}>
<option value="">Select a competitor…</option>
{(hotels || []).map(h => (
<option key={h.hotel_id} value={h.hotel_id}>{h.hotel_name}</option>
))}
</select>
</div>
</div>
</div>
{selectedHotel && analysisLoading && (
<div className="loading-state"><div className="spinner" />Loading analysis…</div>
)}
{selectedHotel && !analysisLoading && analysis && (
<div style={{ display: 'flex', flexDirection: 'column', gap: 20 }}>
{/* Strategy card */}
<div className="card">
<div className="card-header">Pricing Strategy</div>
<div className="card-body" style={{ display: 'flex', gap: 32, flexWrap: 'wrap' }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 10 }}>
{strategyIcon(analysis.strategy.label)}
<span style={{ fontWeight: 700, fontSize: 15 }}>{analysis.strategy.label}</span>
</div>
<StatChip label="Advance Discount" value={`${analysis.strategy.advance_discount_pct.toFixed(1)}%`}
hint="price delta from 90→7 days ahead" />
<StatChip label="Weekend Premium" value={`${analysis.strategy.weekend_premium_pct.toFixed(1)}%`}
hint="Fri-Sun vs Mon-Thu" />
<StatChip label="Sold-Out Rate" value={`${analysis.strategy.avg_sold_out_rate_pct.toFixed(1)}%`}
hint="% of scraped dates with no availability" />
{analysis.strategy.peak_months.length > 0 && (
<StatChip label="Peak Months" value={analysis.strategy.peak_months.join(', ')} hint="" />
)}
</div>
</div>
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 20 }}>
{/* Advance purchase curve */}
<div className="card">
<div className="card-header">Advance Purchase Curve</div>
<div className="card-body" style={{ height: 260 }}>
{analysis.advance_curve.length > 0 ? (
<Plot
style={{ width: '100%', height: '100%' }}
data={[{
type: 'scatter',
mode: 'lines+markers',
x: analysis.advance_curve.map(p => p.days_ahead),
y: analysis.advance_curve.map(p => p.avg_price),
line: { color: '#c9a84c', width: 2 },
marker: { size: 4, color: '#c9a84c' },
hovertemplate: '%{x} days ahead: £%{y:.2f}<extra></extra>',
}]}
layout={{
...PLOT_LAYOUT_BASE,
xaxis: { ...PLOT_LAYOUT_BASE.xaxis, title: { text: 'Days ahead', font: { size: 11 } }, autorange: 'reversed' },
yaxis: { ...PLOT_LAYOUT_BASE.yaxis, title: { text: 'Avg price (£)', font: { size: 11 } }, tickprefix: '£' },
}}
config={{ displayModeBar: false, responsive: true }}
useResizeHandler
/>
) : (
<div className="empty-state">Not enough data yet.</div>
)}
</div>
</div>
{/* DOW breakdown */}
<div className="card">
<div className="card-header">Day-of-Week Breakdown</div>
<div className="card-body" style={{ height: 260 }}>
{analysis.dow_breakdown.length > 0 ? (
<Plot
style={{ width: '100%', height: '100%' }}
data={[{
type: 'bar',
x: analysis.dow_breakdown.map(d => d.dow_name),
y: analysis.dow_breakdown.map(d => d.avg_price),
marker: {
color: analysis.dow_breakdown.map(d =>
d.dow >= 5 ? '#c9a84c' : '#3b82f6'
),
},
hovertemplate: '%{x}: £%{y:.2f}<extra></extra>',
}]}
layout={{
...PLOT_LAYOUT_BASE,
yaxis: { ...PLOT_LAYOUT_BASE.yaxis, tickprefix: '£' },
}}
config={{ displayModeBar: false, responsive: true }}
useResizeHandler
/>
) : (
<div className="empty-state">Not enough data yet.</div>
)}
</div>
</div>
</div>
{/* Rate timeline */}
<div className="card">
<div className="card-header" style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<span>Rate Timeline — How Rates Changed for One Date</span>
<input type="date" style={{ width: 140 }} value={timelineDate}
onChange={e => setTimelineDate(e.target.value)} />
</div>
<div className="card-body" style={{ height: 280 }}>
{(timeline || []).length === 0 ? (
<div className="empty-state">No timeline data for this date.</div>
) : (
<Plot
style={{ width: '100%', height: '100%' }}
data={buildTimelineTraces(timeline || [])}
layout={{
...PLOT_LAYOUT_BASE,
showlegend: true,
legend: { font: { size: 11 }, bgcolor: 'transparent' },
margin: { t: 20, r: 120, b: 48, l: 56 },
xaxis: { ...PLOT_LAYOUT_BASE.xaxis },
yaxis: { ...PLOT_LAYOUT_BASE.yaxis, tickprefix: '£' },
}}
config={{ displayModeBar: false, responsive: true }}
useResizeHandler
/>
)}
</div>
</div>
</div>
)}
{!selectedHotel && (
<div className="empty-state" style={{ marginTop: 0 }}>
Select a competitor above to view their pricing strategy and advance purchase curve.
</div>
)}
</div>
)
}
function StatChip({ label, value, hint }: { label: string; value: string; hint: string }) {
return (
<div style={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
<span style={{ fontSize: 11, color: 'var(--text-mid)', fontWeight: 500, textTransform: 'uppercase', letterSpacing: '0.04em' }}>{label}</span>
<span style={{ fontSize: 18, fontWeight: 700, color: 'var(--text-dark)', lineHeight: 1 }}>{value}</span>
{hint && <span style={{ fontSize: 11, color: 'var(--text-mid)' }}>{hint}</span>}
</div>
)
}
function buildTimelineTraces(entries: TimelineEntry[]) {
const byRoom: Record<string, TimelineEntry[]> = {}
for (const e of entries) {
const key = e.room_label || e.room_id
if (!byRoom[key]) byRoom[key] = []
byRoom[key].push(e)
}
const colors = ['#c9a84c', '#3b82f6', '#10b981', '#ef4444', '#8b5cf6', '#f59e0b']
return Object.entries(byRoom).map(([room, pts], i) => ({
type: 'scatter' as const,
mode: 'lines+markers' as const,
name: room,
x: pts.map(p => p.scraped_at),
y: pts.map(p => p.price_incl),
line: { color: colors[i % colors.length], width: 2 },
marker: { size: 5, color: colors[i % colors.length] },
hovertemplate: `${room}: £%{y:.2f}<extra></extra>`,
}))
}

224
frontend/src/pages/Settings.tsx Normal file
View file

@ -0,0 +1,224 @@
import { useState } from 'react'
import { useParams, useNavigate } from 'react-router-dom'
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
import { Save, RefreshCw, Database, Clock } from 'lucide-react'
import api from '../api'
const TABS = [
{ id: 'newbook', label: 'Newbook Sync' },
{ id: 'system', label: 'System' },
]
interface SystemConfig {
[key: string]: string | null
}
export default function Settings() {
const { tab: tabParam } = useParams<{ tab?: string }>()
const navigate = useNavigate()
const qc = useQueryClient()
const activeTab = tabParam || 'newbook'
const { data: config, isLoading } = useQuery<SystemConfig>({
queryKey: ['system-config'],
queryFn: () => api.get('/competitors/config/system').then(r => r.data),
})
const saveMutation = useMutation({
mutationFn: (payload: { key: string; value: string }) =>
api.post('/competitors/config/system', payload),
onSuccess: () => qc.invalidateQueries({ queryKey: ['system-config'] }),
})
const syncNow = useMutation({
mutationFn: () => api.post('/bookability/refresh-all'),
onSuccess: () => qc.invalidateQueries({ queryKey: ['system-config'] }),
})
return (
<div>
<div className="page-header">
<div>
<div className="page-title">Settings</div>
<div className="page-subtitle">Newbook sync and system configuration</div>
</div>
</div>
<div className="sub-nav">
{TABS.map(t => (
<button
key={t.id}
className={`sub-nav-item${activeTab === t.id ? ' active' : ''}`}
onClick={() => navigate(`/settings/${t.id}`)}
>
{t.label}
</button>
))}
</div>
{activeTab === 'newbook' && (
<NewbookTab
config={config}
isLoading={isLoading}
onSave={(key, val) => saveMutation.mutate({ key, value: val })}
onSyncNow={() => syncNow.mutate()}
saving={saveMutation.isPending}
syncing={syncNow.isPending}
/>
)}
{activeTab === 'system' && (
<SystemTab config={config} isLoading={isLoading} />
)}
</div>
)
}
// ─── Newbook Sync Tab ─────────────────────────────────────────────────────────
interface NewbookTabProps {
config: SystemConfig | undefined
isLoading: boolean
onSave: (key: string, val: string) => void
onSyncNow: () => void
saving: boolean
syncing: boolean
}
function NewbookTab({ config, isLoading, onSave, onSyncNow, saving, syncing }: NewbookTabProps) {
const [syncTime, setSyncTime] = useState('')
const syncEnabled = config?.sync_newbook_current_rates_enabled === 'true'
const currentTime = config?.sync_newbook_current_rates_time || '05:20'
if (isLoading) {
return <div className="loading-state"><div className="spinner" /> Loading…</div>
}
return (
<div style={{ display: 'flex', flexDirection: 'column', gap: 20, maxWidth: 600 }}>
<div className="card">
<div className="card-header">
<span style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<Database size={16} strokeWidth={1.75} />
Newbook Rates Sync
</span>
<span className={`badge ${syncEnabled ? 'badge-success' : 'badge-neutral'}`}>
{syncEnabled ? 'Enabled' : 'Disabled'}
</span>
</div>
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
<p style={{ fontSize: 13, color: 'var(--text-mid)', margin: 0 }}>
When enabled, the app fetches current tariff rates from the Newbook API daily and
stores them for the Bookability view and rate parity calculations.
</p>
<div style={{ display: 'flex', gap: 12 }}>
<button
className={`btn ${syncEnabled ? 'btn-outline' : 'btn-primary'}`}
onClick={() => onSave('sync_newbook_current_rates_enabled', syncEnabled ? 'false' : 'true')}
disabled={saving}
>
{syncEnabled ? 'Disable Sync' : 'Enable Sync'}
</button>
<button
className="btn btn-outline"
onClick={onSyncNow}
disabled={syncing}
>
<RefreshCw size={14} strokeWidth={1.75} />
{syncing ? 'Refreshing…' : 'Sync Now'}
</button>
</div>
</div>
</div>
<div className="card">
<div className="card-header">
<span style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<Clock size={16} strokeWidth={1.75} />
Sync Schedule
</span>
</div>
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 12 }}>
<div>
<label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 6 }}>
Daily sync time (HH:MM)
</label>
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
<input
type="time"
style={{ width: 130 }}
defaultValue={currentTime}
onChange={e => setSyncTime(e.target.value)}
/>
<button
className="btn btn-primary btn-sm"
onClick={() => syncTime && onSave('sync_newbook_current_rates_time', syncTime)}
disabled={saving || !syncTime}
>
<Save size={13} strokeWidth={1.75} />
Save
</button>
</div>
<p style={{ fontSize: 12, color: 'var(--text-mid)', marginTop: 6 }}>
Current: {currentTime} — Booking.com scraper runs at {config?.booking_scraper_daily_time || '05:30'}
</p>
</div>
</div>
</div>
</div>
)
}
// ─── System Tab ───────────────────────────────────────────────────────────────
function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; isLoading: boolean }) {
if (isLoading) {
return <div className="loading-state"><div className="spinner" /> Loading…</div>
}
const displayKeys = [
'booking_scraper_enabled',
'booking_scraper_paused',
'booking_scraper_backend',
'booking_scraper_daily_time',
'sync_newbook_current_rates_enabled',
'sync_newbook_current_rates_time',
]
return (
<div style={{ maxWidth: 700 }}>
<div className="card">
<div className="card-header">System Configuration</div>
<div className="table-wrap">
<table>
<thead>
<tr>
<th>Key</th>
<th>Value</th>
</tr>
</thead>
<tbody>
{displayKeys.map(k => (
<tr key={k}>
<td><code style={{ fontSize: 12, color: 'var(--text-mid)' }}>{k}</code></td>
<td>
<span style={{ fontSize: 13 }}>
{config?.[k] ?? <em style={{ color: 'var(--text-mid)' }}>not set</em>}
</span>
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
<div style={{ marginTop: 16, fontSize: 12, color: 'var(--text-mid)' }}>
To configure the Booking.com scraper location and hotel tiers, use the Settings tab inside{' '}
<a href="/rates/market" style={{ color: 'var(--gold)' }}>Market View</a>.
</div>
</div>
)
}

114
frontend/src/types.ts Normal file
View file

@ -0,0 +1,114 @@
export interface User {
email: string
name: string
is_admin: boolean
caps: string[]
}
export function can(user: User | null, cap: string): boolean {
if (!user) return false
return user.is_admin || user.caps.includes(cap)
}
export interface Hotel {
id: number
name: string
tier: 'own' | 'competitor' | 'market'
star_rating: number | null
review_score: number | null
booking_com_url: string | null
booking_com_id: string | null
display_order: number
notes: string | null
is_active: boolean
scraped_dates?: number
last_scraped?: string | null
}
export interface RateMatrixEntry {
hotel_id: number
hotel_name: string
tier: string
rate_date: string
rate_gross: number | null
availability_status: string
rooms_left: number | null
scraped_at: string | null
}
export interface ScraperStatus {
enabled: boolean
paused: boolean
pause_until: string | null
backend: string
daily_time: string
last_batch: {
batch_id: string
started_at: string
status: string
rates_scraped: number
dates_completed: number
} | null
}
export interface BookabilityCategory {
category_id: string
category_name: string
room_count: number
display_order: number
dates: BookabilityDate[]
}
export interface BookabilityDate {
date: string
gross_rate: number | null
tariffs: TariffSummary[]
occupancy_pct: number | null
available_rooms: number | null
is_bookable: boolean
}
export interface TariffSummary {
tariff_name: string
rate: number | null
min_stay: number | null
advance_max: number | null
is_available: boolean
}
export interface AdvancePurchaseCurvePoint {
lead_bucket: string
avg_rate: number
sample_count: number
}
export interface DowAnalysisPoint {
dow: number
dow_label: string
avg_rate: number
date_count: number
}
export interface StrategySummary {
advance_discount_pct: number | null
weekend_premium_pct: number | null
avg_sold_out_rate_pct: number | null
strategy_label: string
}
export interface HotelAnalysis {
hotel: Hotel
date_range: { from: string; to: string }
advance_purchase_curve: AdvancePurchaseCurvePoint[]
dow_analysis: DowAnalysisPoint[]
sold_out_pattern: Array<{ dow: number; dow_label: string; total_dates: number; sold_out_dates: number }>
strategy_summary: StrategySummary
}
export interface RateTimelinePoint {
scraped_at: string
rate_gross: number | null
availability_status: string
rooms_left: number | null
days_out: number
}

7
frontend/vite.config.ts Normal file
View file

@ -0,0 +1,7 @@
import { defineConfig } from 'vite'
import react from '@vitejs/plugin-react'
export default defineConfig({
plugins: [react()],
base: '/rates/',
})