From e05054172f3248bd9dadcafb359d4ff0126e1f1a Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Sun, 5 Jul 2026 12:06:30 +0000 Subject: [PATCH] =?UTF-8?q?Add=20Rate=20Monitor=20app=20=E2=80=94=20Bookin?= =?UTF-8?q?g.com=20+=20direct=20booking=20engine=20competitor=20rates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combines Booking.com Playwright scraper (from forecasting), direct booking engine scraper (ported from laptop-archive/guestline-monitor), and Newbook own-hotel rates into one focused tool. Four views: Bookability, Market View (with price index badges + direct rate sub-rows), Direct Rates (per-competitor room breakdown, min-stay flags, hotel config/discovery), Rate Analysis (advance purchase curve, DOW chart, rate timeline, comparison table). Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 13 + backend/Dockerfile | 22 + backend/api/__init__.py | 0 backend/api/analysis.py | 317 +++ backend/api/bookability.py | 624 ++++++ backend/api/competitors.py | 940 ++++++++ backend/api/direct.py | 337 +++ backend/auth.py | 69 + backend/database.py | 37 + backend/jobs/__init__.py | 0 backend/jobs/fetch_current_rates.py | 370 ++++ backend/jobs/scrape_booking_rates.py | 165 ++ backend/jobs/scrape_direct_rates.py | 43 + backend/main.py | 80 + backend/migrate_direct_data.py | 167 ++ backend/requirements.txt | 14 + backend/scheduler.py | 120 + backend/schema.sql | 267 +++ backend/services/__init__.py | 0 backend/services/booking_scraper.py | 829 +++++++ backend/services/central_settings.py | 100 + backend/services/direct_profiles/__init__.py | 28 + backend/services/direct_profiles/base.py | 25 + .../services/direct_profiles/directbook.py | 175 ++ backend/services/direct_profiles/guestline.py | 49 + .../direct_profiles/newbook_scrape.py | 235 ++ .../services/direct_profiles/travelclick.py | 166 ++ backend/services/direct_scraper.py | 295 +++ backend/services/newbook_rates_client.py | 863 ++++++++ backend/services/scraper_backends/__init__.py | 20 + backend/services/scraper_backends/base.py | 152 ++ .../scraper_backends/playwright_local.py | 401 ++++ docker-compose.yml | 36 + frontend/Dockerfile | 15 + frontend/index.html | 12 + frontend/nginx.conf | 35 + frontend/package.json | 30 + frontend/src/App.tsx | 28 + frontend/src/api.ts | 20 + frontend/src/components/AuthGate.tsx | 47 + frontend/src/components/Layout.tsx | 70 + frontend/src/index.css | 363 +++ frontend/src/main.tsx | 20 + frontend/src/pages/Bookability.tsx | 1094 ++++++++++ frontend/src/pages/DirectRates.tsx | 520 +++++ frontend/src/pages/MarketView.tsx | 1940 +++++++++++++++++ frontend/src/pages/RateAnalysis.tsx | 362 +++ frontend/src/pages/Settings.tsx | 224 ++ frontend/src/types.ts | 114 + frontend/vite.config.ts | 7 + 50 files changed, 11860 insertions(+) create mode 100644 .gitignore create mode 100644 backend/Dockerfile create mode 100644 backend/api/__init__.py create mode 100644 backend/api/analysis.py create mode 100644 backend/api/bookability.py create mode 100644 backend/api/competitors.py create mode 100644 backend/api/direct.py create mode 100644 backend/auth.py create mode 100644 backend/database.py create mode 100644 backend/jobs/__init__.py create mode 100644 backend/jobs/fetch_current_rates.py create mode 100644 backend/jobs/scrape_booking_rates.py create mode 100644 backend/jobs/scrape_direct_rates.py create mode 100644 backend/main.py create mode 100644 backend/migrate_direct_data.py create mode 100644 backend/requirements.txt create mode 100644 backend/scheduler.py create mode 100644 backend/schema.sql create mode 100644 backend/services/__init__.py create mode 100644 backend/services/booking_scraper.py create mode 100644 backend/services/central_settings.py create mode 100644 backend/services/direct_profiles/__init__.py create mode 100644 backend/services/direct_profiles/base.py create mode 100644 backend/services/direct_profiles/directbook.py create mode 100644 backend/services/direct_profiles/guestline.py create mode 100644 backend/services/direct_profiles/newbook_scrape.py create mode 100644 backend/services/direct_profiles/travelclick.py create mode 100644 backend/services/direct_scraper.py create mode 100644 backend/services/newbook_rates_client.py create mode 100644 backend/services/scraper_backends/__init__.py create mode 100644 backend/services/scraper_backends/base.py create mode 100644 backend/services/scraper_backends/playwright_local.py create mode 100644 docker-compose.yml create mode 100644 frontend/Dockerfile create mode 100644 frontend/index.html create mode 100644 frontend/nginx.conf create mode 100644 frontend/package.json create mode 100644 frontend/src/App.tsx create mode 100644 frontend/src/api.ts create mode 100644 frontend/src/components/AuthGate.tsx create mode 100644 frontend/src/components/Layout.tsx create mode 100644 frontend/src/index.css create mode 100644 frontend/src/main.tsx create mode 100644 frontend/src/pages/Bookability.tsx create mode 100644 frontend/src/pages/DirectRates.tsx create mode 100644 frontend/src/pages/MarketView.tsx create mode 100644 frontend/src/pages/RateAnalysis.tsx create mode 100644 frontend/src/pages/Settings.tsx create mode 100644 frontend/src/types.ts create mode 100644 frontend/vite.config.ts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c89421 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +node_modules/ +dist/ +__pycache__/ +*.pyc +*.pyo +.env +.env.local +.DS_Store +*.egg-info/ +.eggs/ +build/ +.venv/ +venv/ diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..0d810ed --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + build-essential \ + libpq-dev \ + postgresql-client \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Playwright + Chromium for Booking.com scraper +RUN playwright install chromium --with-deps + +COPY . . + +EXPOSE 8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/api/__init__.py b/backend/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/api/analysis.py b/backend/api/analysis.py new file mode 100644 index 0000000..621cbca --- /dev/null +++ b/backend/api/analysis.py @@ -0,0 +1,317 @@ +""" +Rate Analysis API — advance purchase curves, DOW analysis, rate timelines, strategy summary +""" +from fastapi import APIRouter, Depends, Query +from sqlalchemy import text +from typing import Optional +from datetime import date, timedelta + +from database import AsyncSessionLocal +from auth import get_current_user, require_cap + +router = APIRouter() + + +async def get_db(): + async with AsyncSessionLocal() as db: + yield db + + +# ─── Hotels available for analysis ─────────────────────────────────────────── + +@router.get("/hotels") +async def list_analysis_hotels( + tier: Optional[str] = Query(None, description="Filter by tier: own|competitor|market"), + user=Depends(get_current_user) +): + require_cap(user, "rate_analysis") + async with AsyncSessionLocal() as db: + where = "is_active = true" + params = {} + if tier: + where += " AND tier = :tier" + params["tier"] = tier + result = await db.execute( + text(f""" + SELECT h.id, h.name, h.tier, h.star_rating, h.review_score, + h.booking_com_url, + COUNT(DISTINCT r.rate_date) AS scraped_dates, + MAX(r.scraped_at) AS last_scraped + FROM booking_com_hotels h + LEFT JOIN booking_com_rates r ON r.hotel_id = h.id + WHERE {where} + GROUP BY h.id, h.name, h.tier, h.star_rating, h.review_score, h.booking_com_url + ORDER BY h.tier, h.display_order, h.name + """), + params + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + +# ─── Full analysis for one hotel ───────────────────────────────────────────── + +@router.get("/hotel/{hotel_id}") +async def analyse_hotel( + hotel_id: int, + from_date: date = Query(default_factory=lambda: date.today()), + to_date: date = Query(default_factory=lambda: date.today() + timedelta(days=89)), + user=Depends(get_current_user) +): + require_cap(user, "rate_analysis") + async with AsyncSessionLocal() as db: + # Hotel info + hotel_row = await db.execute( + text("SELECT id, name, tier, star_rating, booking_com_url FROM booking_com_hotels WHERE id = :id"), + {"id": hotel_id} + ) + hotel = hotel_row.mappings().fetchone() + if not hotel: + from fastapi import HTTPException + raise HTTPException(status_code=404, detail="Hotel not found") + + # Advance purchase curve + apc_result = await db.execute( + text(""" + SELECT + CASE + WHEN (rate_date - scraped_at::date) <= 7 THEN '0-7d' + WHEN (rate_date - scraped_at::date) <= 29 THEN '8-29d' + WHEN (rate_date - scraped_at::date) <= 89 THEN '30-89d' + ELSE '90+d' + END AS lead_bucket, + ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate, + COUNT(*) AS sample_count + FROM booking_com_rates + WHERE hotel_id = :hid + AND rate_date BETWEEN :from_date AND :to_date + AND availability_status = 'available' + AND rate_gross IS NOT NULL + GROUP BY lead_bucket + ORDER BY + CASE lead_bucket + WHEN '0-7d' THEN 1 + WHEN '8-29d' THEN 2 + WHEN '30-89d' THEN 3 + ELSE 4 + END + """), + {"hid": hotel_id, "from_date": from_date, "to_date": to_date} + ) + advance_purchase_curve = [dict(r) for r in apc_result.mappings().all()] + + # Day-of-week averages (latest scrape per date) + dow_result = await db.execute( + text(""" + SELECT + EXTRACT(DOW FROM rate_date)::int AS dow, + TO_CHAR(rate_date, 'Dy') AS dow_label, + ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate, + COUNT(DISTINCT rate_date) AS date_count + FROM ( + SELECT DISTINCT ON (rate_date) + rate_date, rate_gross + FROM booking_com_rates + WHERE hotel_id = :hid + AND rate_date BETWEEN :from_date AND :to_date + AND availability_status = 'available' + AND rate_gross IS NOT NULL + ORDER BY rate_date, scraped_at DESC + ) latest + GROUP BY dow, dow_label + ORDER BY dow + """), + {"hid": hotel_id, "from_date": from_date, "to_date": to_date} + ) + dow_analysis = [dict(r) for r in dow_result.mappings().all()] + + # Sold-out pattern by day-of-week + sold_out_result = await db.execute( + text(""" + SELECT + EXTRACT(DOW FROM rate_date)::int AS dow, + TO_CHAR(rate_date, 'Dy') AS dow_label, + COUNT(DISTINCT rate_date) AS total_dates, + COUNT(DISTINCT rate_date) FILTER ( + WHERE availability_status = 'sold_out' + ) AS sold_out_dates + FROM ( + SELECT DISTINCT ON (rate_date) + rate_date, availability_status + FROM booking_com_rates + WHERE hotel_id = :hid + AND rate_date BETWEEN :from_date AND :to_date + ORDER BY rate_date, scraped_at DESC + ) latest + GROUP BY dow, dow_label + ORDER BY dow + """), + {"hid": hotel_id, "from_date": from_date, "to_date": to_date} + ) + sold_out_pattern = [dict(r) for r in sold_out_result.mappings().all()] + + # Strategy summary + strategy = _compute_strategy(advance_purchase_curve, dow_analysis, sold_out_pattern) + + return { + "hotel": dict(hotel), + "date_range": {"from": str(from_date), "to": str(to_date)}, + "advance_purchase_curve": advance_purchase_curve, + "dow_analysis": dow_analysis, + "sold_out_pattern": sold_out_pattern, + "strategy_summary": strategy, + } + + +# ─── Rate timeline for a single date ───────────────────────────────────────── + +@router.get("/hotel/{hotel_id}/timeline") +async def rate_timeline( + hotel_id: int, + rate_date: date = Query(...), + user=Depends(get_current_user) +): + require_cap(user, "rate_analysis") + async with AsyncSessionLocal() as db: + result = await db.execute( + text(""" + SELECT + scraped_at, + rate_gross, + availability_status, + rooms_left, + room_type, + (rate_date - scraped_at::date) AS days_out + FROM booking_com_rates + WHERE hotel_id = :hid AND rate_date = :rd + ORDER BY scraped_at ASC + """), + {"hid": hotel_id, "rd": rate_date} + ) + rows = result.mappings().all() + return { + "hotel_id": hotel_id, + "rate_date": str(rate_date), + "timeline": [dict(r) for r in rows], + } + + +# ─── Own vs Competitor comparison ──────────────────────────────────────────── + +@router.get("/comparison") +async def rate_comparison( + competitor_ids: str = Query(..., description="Comma-separated hotel IDs to compare"), + from_date: date = Query(default_factory=lambda: date.today()), + to_date: date = Query(default_factory=lambda: date.today() + timedelta(days=29)), + user=Depends(get_current_user) +): + require_cap(user, "rate_analysis") + try: + comp_ids = [int(x.strip()) for x in competitor_ids.split(",") if x.strip()] + except ValueError: + from fastapi import HTTPException + raise HTTPException(status_code=400, detail="competitor_ids must be comma-separated integers") + + async with AsyncSessionLocal() as db: + # Own hotel latest rates + own_result = await db.execute( + text(""" + SELECT + ncr.rate_date, + ncr.gross_rate AS own_rate + FROM newbook_current_rates ncr + WHERE ncr.rate_date BETWEEN :from_date AND :to_date + ORDER BY ncr.rate_date + """), + {"from_date": from_date, "to_date": to_date} + ) + own_rates = {str(r.rate_date): float(r.own_rate) for r in own_result if r.own_rate} + + # Competitor latest rates per date + comp_result = await db.execute( + text(""" + SELECT + r.hotel_id, + h.name AS hotel_name, + r.rate_date, + r.rate_gross, + r.availability_status + FROM ( + SELECT DISTINCT ON (hotel_id, rate_date) + hotel_id, rate_date, rate_gross, availability_status + FROM booking_com_rates + WHERE hotel_id = ANY(:comp_ids) + AND rate_date BETWEEN :from_date AND :to_date + ORDER BY hotel_id, rate_date, scraped_at DESC + ) r + JOIN booking_com_hotels h ON h.id = r.hotel_id + ORDER BY r.rate_date, h.display_order + """), + {"comp_ids": comp_ids, "from_date": from_date, "to_date": to_date} + ) + comp_rows = comp_result.mappings().all() + + # Build per-date rows + date_map: dict = {} + hotel_names: dict = {} + for row in comp_rows: + d = str(row["rate_date"]) + if d not in date_map: + date_map[d] = {"date": d, "own_rate": own_rates.get(d)} + date_map[d][f"h{row['hotel_id']}"] = float(row["rate_gross"]) if row["rate_gross"] else None + date_map[d][f"h{row['hotel_id']}_status"] = row["availability_status"] + hotel_names[row["hotel_id"]] = row["hotel_name"] + + return { + "hotel_names": hotel_names, + "rows": sorted(date_map.values(), key=lambda x: x["date"]), + } + + +# ─── Strategy computation helper ───────────────────────────────────────────── + +def _compute_strategy(apc: list, dow: list, sold_out: list) -> dict: + # Advance discount — compare 0-7d vs 30-89d + rates_by_bucket = {r["lead_bucket"]: float(r["avg_rate"]) for r in apc if r.get("avg_rate")} + advance_discount_pct = None + if "0-7d" in rates_by_bucket and "30-89d" in rates_by_bucket: + close_in = rates_by_bucket["0-7d"] + far_out = rates_by_bucket["30-89d"] + if far_out > 0: + # Positive = closes in higher (scarcity premium); negative = discount for advance + advance_discount_pct = round((close_in - far_out) / far_out * 100, 1) + + # Weekend premium — Fri(5)+Sat(6) vs Mon(1)–Thu(4) + rates_by_dow = {r["dow"]: float(r["avg_rate"]) for r in dow if r.get("avg_rate")} + weekend_premium_pct = None + weekend_rates = [rates_by_dow[d] for d in [5, 6] if d in rates_by_dow] + weekday_rates = [rates_by_dow[d] for d in [1, 2, 3, 4] if d in rates_by_dow] + if weekend_rates and weekday_rates: + avg_wk = sum(weekend_rates) / len(weekend_rates) + avg_wd = sum(weekday_rates) / len(weekday_rates) + if avg_wd > 0: + weekend_premium_pct = round((avg_wk - avg_wd) / avg_wd * 100, 1) + + # Sold-out rate + total_dates = sum(r["total_dates"] for r in sold_out) + sold_out_dates = sum(r["sold_out_dates"] for r in sold_out) + sold_out_rate_pct = round(sold_out_dates / total_dates * 100, 1) if total_dates > 0 else None + + # Strategy label + label = "Mixed / insufficient data" + if advance_discount_pct is not None: + if advance_discount_pct <= -5: + label = "Advance-booking discounter" + elif advance_discount_pct >= 8 and (sold_out_rate_pct or 0) >= 10: + label = "Yield manager (scarcity-driven)" + elif advance_discount_pct >= 3: + label = "Flat-rate / hold-firm strategy" + else: + label = "Stable pricing" + + return { + "advance_discount_pct": advance_discount_pct, + "weekend_premium_pct": weekend_premium_pct, + "avg_sold_out_rate_pct": sold_out_rate_pct, + "strategy_label": label, + } diff --git a/backend/api/bookability.py b/backend/api/bookability.py new file mode 100644 index 0000000..1d741f7 --- /dev/null +++ b/backend/api/bookability.py @@ -0,0 +1,624 @@ +""" +Bookability API endpoints +Rate availability matrix and competitor rate comparison +""" +from typing import Optional, List, Dict, Any +from datetime import date, datetime, timedelta +from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import text +from pydantic import BaseModel +import logging +import json + +from database import get_db, SyncSessionLocal +from auth import get_current_user + +router = APIRouter() +logger = logging.getLogger(__name__) + + +# ============================================ +# RESPONSE MODELS +# ============================================ + +class CategoryInfo(BaseModel): + category_id: str + category_name: str + room_count: int + + +class TariffInfo(BaseModel): + name: str + description: Optional[str] = None + rate: Optional[float] = None + average_nightly: Optional[float] = None + available: bool + message: str + sort_order: int = 999 + min_stay: Optional[int] = None + available_for_min_stay: Optional[bool] = None # True if available when queried with min_stay nights + + +class OccupancyInfo(BaseModel): + occupied: int = 0 + available: int = 0 + maintenance: int = 0 + + +class DateRateInfo(BaseModel): + rate_gross: Optional[float] = None + rate_net: Optional[float] = None + tariffs: List[TariffInfo] + tariff_count: int + occupancy: Optional[OccupancyInfo] = None + valid_from: Optional[str] = None + + +class RateMatrixResponse(BaseModel): + categories: List[CategoryInfo] + dates: List[str] + matrix: Dict[str, Dict[str, DateRateInfo]] + date_last_updated: Dict[str, Optional[str]] = {} + + +# ============================================ +# RATE MATRIX ENDPOINT +# ============================================ + +@router.get("/rate-matrix", response_model=RateMatrixResponse) +async def get_rate_matrix( + from_date: Optional[str] = None, + to_date: Optional[str] = None, + category_id: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get rate availability matrix for all tariffs across dates and categories. + + Returns a matrix showing all available tariff options for each room category + and date combination, including availability status and rates. + + Args: + from_date: Start date (YYYY-MM-DD), defaults to today + to_date: End date (YYYY-MM-DD), defaults to today + 30 days + category_id: Optional filter to specific category + + Returns: + RateMatrixResponse with categories, dates, and the matrix data + """ + # Default date range + today = date.today() + start = date.fromisoformat(from_date) if from_date else today + end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30) + + # Validate date range + if end < start: + raise HTTPException(status_code=400, detail="to_date must be after from_date") + if (end - start).days > 366: + raise HTTPException(status_code=400, detail="Date range cannot exceed 366 days") + + # Fetch categories + cat_query = """ + SELECT site_id, site_name, room_count + FROM newbook_room_categories + WHERE is_included = true + """ + params: Dict[str, Any] = {} + + if category_id: + cat_query += " AND site_id = :category_id" + params["category_id"] = category_id + + cat_query += " ORDER BY display_order, site_name" + + cat_result = await db.execute(text(cat_query), params) + categories = [ + CategoryInfo( + category_id=row.site_id, + category_name=row.site_name, + room_count=row.room_count or 0 + ) + for row in cat_result.fetchall() + ] + + if not categories: + return RateMatrixResponse(categories=[], dates=[], matrix={}) + + # Build date list + dates = [] + current = start + while current <= end: + dates.append(current.isoformat()) + current += timedelta(days=1) + + # Fetch rates with tariffs_data (get latest version per category/date) + rates_query = """ + SELECT DISTINCT ON (category_id, rate_date) + category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from + FROM newbook_current_rates + WHERE rate_date >= :from_date AND rate_date <= :to_date + """ + rates_params: Dict[str, Any] = {"from_date": start, "to_date": end} + + if category_id: + rates_query += " AND category_id = :category_id" + rates_params["category_id"] = category_id + + rates_query += " ORDER BY category_id, rate_date, valid_from DESC" + + rates_result = await db.execute(text(rates_query), rates_params) + rates_rows = rates_result.fetchall() + + # Fetch occupancy data from newbook_occupancy_report_data + occupancy_query = """ + SELECT category_id, date, occupied, available, maintenance + FROM newbook_occupancy_report_data + WHERE date >= :from_date AND date <= :to_date + """ + occupancy_params: Dict[str, Any] = {"from_date": start, "to_date": end} + + if category_id: + occupancy_query += " AND category_id = :category_id" + occupancy_params["category_id"] = category_id + + occupancy_result = await db.execute(text(occupancy_query), occupancy_params) + occupancy_rows = occupancy_result.fetchall() + + # Build occupancy lookup: category_id -> date -> OccupancyInfo + occupancy_map: Dict[str, Dict[str, OccupancyInfo]] = {} + for row in occupancy_rows: + cat_id = row.category_id + occ_date = row.date.isoformat() + if cat_id not in occupancy_map: + occupancy_map[cat_id] = {} + occupancy_map[cat_id][occ_date] = OccupancyInfo( + occupied=row.occupied or 0, + available=row.available or 0, + maintenance=row.maintenance or 0 + ) + + # Build matrix + matrix: Dict[str, Dict[str, DateRateInfo]] = {} + + # Initialize matrix with empty data for all categories and dates + for cat in categories: + matrix[cat.category_id] = {} + for date_str in dates: + # Get occupancy for this category/date if available + occ = occupancy_map.get(cat.category_id, {}).get(date_str) + matrix[cat.category_id][date_str] = DateRateInfo( + rate_gross=None, + rate_net=None, + tariffs=[], + tariff_count=0, + occupancy=occ + ) + + # Populate matrix with actual data + for row in rates_rows: + cat_id = row.category_id + rate_date = row.rate_date.isoformat() + + if cat_id not in matrix or rate_date not in matrix[cat_id]: + continue + + # Parse tariffs_data + tariffs_data = row.tariffs_data or {} + if isinstance(tariffs_data, str): + try: + tariffs_data = json.loads(tariffs_data) + except json.JSONDecodeError: + tariffs_data = {} + + # Build tariff list + tariffs_list = [] + raw_tariffs = tariffs_data.get('tariffs', []) + + for idx, tariff in enumerate(raw_tariffs): + tariffs_list.append(TariffInfo( + name=tariff.get('name', 'Unknown'), + description=tariff.get('description'), + rate=tariff.get('rate'), + average_nightly=tariff.get('average_nightly'), + available=tariff.get('success', False), + message=tariff.get('message', ''), + sort_order=tariff.get('sort_order', idx), + min_stay=tariff.get('min_stay'), + available_for_min_stay=tariff.get('available_for_min_stay') + )) + + # Preserve existing occupancy data + existing_occ = matrix[cat_id][rate_date].occupancy + vf = row.valid_from.isoformat() if row.valid_from else None + + matrix[cat_id][rate_date] = DateRateInfo( + rate_gross=float(row.rate_gross) if row.rate_gross else None, + rate_net=float(row.rate_net) if row.rate_net else None, + tariffs=tariffs_list, + tariff_count=tariffs_data.get('tariff_count', len(tariffs_list)), + occupancy=existing_occ, + valid_from=vf + ) + + # Per-date latest update time (max valid_from across all categories for each date) + date_last_updated: Dict[str, Optional[str]] = {} + for date_str in dates: + latest = None + for cat in categories: + vf = matrix.get(cat.category_id, {}).get(date_str, DateRateInfo(tariffs=[], tariff_count=0)).valid_from + if vf and (latest is None or vf > latest): + latest = vf + date_last_updated[date_str] = latest + + return RateMatrixResponse( + categories=categories, + dates=dates, + matrix=matrix, + date_last_updated=date_last_updated + ) + + +# ============================================ +# RATE MATRIX SUMMARY (lightweight endpoint) +# ============================================ + +@router.get("/rate-matrix/summary") +async def get_rate_matrix_summary( + from_date: Optional[str] = None, + to_date: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get a summary of rate availability issues. + + Returns counts of unavailable tariffs by category and date for quick + identification of potential bookability problems. + """ + today = date.today() + start = date.fromisoformat(from_date) if from_date else today + end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30) + + # Query rates with issues (get latest version per category/date) + result = await db.execute( + text(""" + SELECT DISTINCT ON (category_id, rate_date) + category_id, + rate_date, + tariffs_data + FROM newbook_current_rates + WHERE rate_date >= :from_date AND rate_date <= :to_date + AND tariffs_data IS NOT NULL + ORDER BY category_id, rate_date, valid_from DESC + """), + {"from_date": start, "to_date": end} + ) + + issues = [] + for row in result.fetchall(): + tariffs_data = row.tariffs_data or {} + if isinstance(tariffs_data, str): + try: + tariffs_data = json.loads(tariffs_data) + except json.JSONDecodeError: + continue + + tariffs = tariffs_data.get('tariffs', []) + unavailable = [t for t in tariffs if not t.get('success', False)] + + if unavailable: + issues.append({ + "category_id": row.category_id, + "date": row.rate_date.isoformat(), + "unavailable_count": len(unavailable), + "unavailable_tariffs": [t.get('name') for t in unavailable], + "messages": [t.get('message') for t in unavailable if t.get('message')] + }) + + return { + "from_date": start.isoformat(), + "to_date": end.isoformat(), + "total_issues": len(issues), + "issues": issues + } + + +# ============================================ +# RATE HISTORY ENDPOINT +# ============================================ + +@router.get("/rate-history/{category_id}/{rate_date}") +async def get_rate_history( + category_id: str, + rate_date: str, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get rate change history for a specific category and date. + + Returns all rate snapshots showing how rates evolved over time. + Useful for understanding when rates changed and by how much. + """ + try: + target_date = date.fromisoformat(rate_date) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD") + + result = await db.execute( + text(""" + SELECT + id, + rate_gross, + rate_net, + tariffs_data, + valid_from, + last_verified_at + FROM newbook_current_rates + WHERE category_id = :category_id AND rate_date = :rate_date + ORDER BY valid_from DESC + """), + {"category_id": category_id, "rate_date": target_date} + ) + + history = [] + for row in result.fetchall(): + tariffs_data = row.tariffs_data or {} + if isinstance(tariffs_data, str): + try: + tariffs_data = json.loads(tariffs_data) + except json.JSONDecodeError: + tariffs_data = {} + + tariffs = tariffs_data.get('tariffs', []) + + history.append({ + "id": row.id, + "rate_gross": float(row.rate_gross) if row.rate_gross else None, + "rate_net": float(row.rate_net) if row.rate_net else None, + "valid_from": row.valid_from.isoformat() if row.valid_from else None, + "last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None, + "tariff_count": len(tariffs), + "tariffs_available": sum(1 for t in tariffs if t.get('success', False)), + "tariffs_unavailable": sum(1 for t in tariffs if not t.get('success', False)), + "tariffs": [ + { + "name": t.get('name'), + "rate": t.get('rate'), + "available": t.get('success', False), + "message": t.get('message', ''), + "min_stay": t.get('min_stay') + } + for t in tariffs + ] + }) + + return { + "category_id": category_id, + "rate_date": rate_date, + "version_count": len(history), + "history": history + } + + +# ============================================ +# RATE CHANGES SUMMARY +# ============================================ + +@router.get("/rate-changes") +async def get_rate_changes( + days: int = 7, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get summary of rate changes in the last N days. + + Shows which rates changed and when, useful for tracking pricing strategy changes. + """ + cutoff = datetime.now() - timedelta(days=days) + + # Find dates with multiple versions (indicating changes) + result = await db.execute( + text(""" + SELECT + category_id, + rate_date, + COUNT(*) as version_count, + MIN(valid_from) as first_version, + MAX(valid_from) as latest_version + FROM newbook_current_rates + WHERE valid_from >= :cutoff + GROUP BY category_id, rate_date + HAVING COUNT(*) > 1 + ORDER BY MAX(valid_from) DESC + LIMIT 100 + """), + {"cutoff": cutoff} + ) + + changes = [] + for row in result.fetchall(): + changes.append({ + "category_id": row.category_id, + "rate_date": row.rate_date.isoformat(), + "version_count": row.version_count, + "first_version": row.first_version.isoformat() if row.first_version else None, + "latest_version": row.latest_version.isoformat() if row.latest_version else None + }) + + return { + "days": days, + "total_changes": len(changes), + "changes": changes + } + + +# ============================================ +# FETCH RATES TRIGGER (manual refresh) +# ============================================ + +@router.post("/refresh-rates") +async def refresh_rates( + from_date: Optional[str] = None, + to_date: Optional[str] = None, + category_id: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Trigger a manual refresh of current rates from Newbook. + + This runs the rate fetch job for the specified date range. + Note: This can be slow as it respects Newbook API rate limits. + """ + from jobs.fetch_current_rates import run_fetch_current_rates + + # For now, just run the standard fetch + # TODO: Add support for custom date range and category filter + try: + await run_fetch_current_rates() + return {"status": "success", "message": "Rates refresh completed"} + except Exception as e: + logger.error(f"Rates refresh failed: {e}") + raise HTTPException(status_code=500, detail=f"Rates refresh failed: {str(e)}") + + +# ============================================ +# SINGLE-DATE RATE REFRESH +# ============================================ + +def _refresh_date_sync(rate_date: date): + """ + Fetch rates for a single date from Newbook and save to DB. + Runs synchronously in a background task. + """ + import asyncio + from decimal import Decimal + from services.newbook_rates_client import NewbookRatesClient + from jobs.fetch_current_rates import save_rate_snapshot + + db = SyncSessionLocal() + try: + # Get config + config_result = db.execute( + text(""" + SELECT config_key, config_value FROM system_config + WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate') + """) + ) + config = {row.config_key: row.config_value for row in config_result.fetchall()} + + # Central Settings service first, app-local config fallback + from services.central_settings import get_newbook_credentials_sync + creds = get_newbook_credentials_sync() + if not creds: + if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']): + logger.error("Newbook credentials not configured for single-date refresh") + return + creds = { + 'api_key': config['newbook_api_key'], + 'username': config['newbook_username'], + 'password': config['newbook_password'], + 'region': config['newbook_region'], + } + + vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20')) + + # Get included categories + cat_result = db.execute( + text("SELECT site_id FROM newbook_room_categories WHERE is_included = true") + ) + included_categories = set(row.site_id for row in cat_result.fetchall()) + + client = NewbookRatesClient( + api_key=creds['api_key'], + username=creds['username'], + password=creds['password'], + region=creds['region'], + vat_rate=vat_rate + ) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + async def _fetch(): + async with client: + # Single-night query for this date (all categories) + category_rates = await client._fetch_all_categories_batch( + rate_date, guests_adults=2, guests_children=0 + ) + + # Check for min_stay tariffs needing multi-night verification + dates_by_nights: Dict[int, list] = {} + for cat_id, rates in category_rates.items(): + if cat_id not in included_categories: + continue + for rate in rates: + for tariff in rate.get('tariffs_data', {}).get('tariffs', []): + min_stay = tariff.get('min_stay') + if min_stay and min_stay > 1 and not tariff.get('success', False): + if min_stay not in dates_by_nights: + dates_by_nights[min_stay] = [] + if rate_date not in dates_by_nights[min_stay]: + dates_by_nights[min_stay].append(rate_date) + + # Run multi-night verification if needed + if dates_by_nights: + multi_results = await client.get_multi_night_availability(dates_by_nights) + for cat_id, rates in category_rates.items(): + if cat_id not in included_categories: + continue + for rate in rates: + if rate_date in multi_results: + cat_avail = multi_results[rate_date].get(cat_id, {}) + for tariff in rate.get('tariffs_data', {}).get('tariffs', []): + if tariff.get('min_stay') and tariff['min_stay'] > 1: + tariff['available_for_min_stay'] = cat_avail.get(tariff.get('name', ''), False) + + # Save snapshots + inserted = 0 + for cat_id, rates in category_rates.items(): + if cat_id not in included_categories: + continue + for rate in rates: + result = save_rate_snapshot(db, cat_id, rate['date'], rate) + if result == 'inserted': + inserted += 1 + + return inserted + + inserted = loop.run_until_complete(_fetch()) + db.commit() + logger.info(f"Single-date refresh for {rate_date}: {inserted} new snapshots") + + finally: + loop.close() + + except Exception as e: + logger.error(f"Single-date refresh failed for {rate_date}: {e}", exc_info=True) + db.rollback() + finally: + db.close() + + +@router.post("/refresh-date/{rate_date}") +async def refresh_single_date( + rate_date: str, + current_user: dict = Depends(get_current_user) +): + """ + Trigger a refresh of rates for a single date from Newbook. + Runs synchronously so the client can invalidate its cache immediately on response. + """ + try: + target_date = date.fromisoformat(rate_date) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD") + + import asyncio + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _refresh_date_sync, target_date) + return {"status": "success", "date": rate_date, "message": f"Rates refreshed for {rate_date}"} diff --git a/backend/api/competitors.py b/backend/api/competitors.py new file mode 100644 index 0000000..12fffa6 --- /dev/null +++ b/backend/api/competitors.py @@ -0,0 +1,940 @@ +""" +Competitor Rates API endpoints +Booking.com rate scraping, hotel management, and competitor comparison +""" +from typing import Optional, List, Dict, Any +from datetime import date, datetime, timedelta +from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import text +from pydantic import BaseModel +import logging + +from database import get_db, SyncSessionLocal +from auth import get_current_user + +router = APIRouter() +logger = logging.getLogger(__name__) + + +# ============================================ +# REQUEST/RESPONSE MODELS +# ============================================ + +class ScrapeRequest(BaseModel): + from_date: str + to_date: Optional[str] = None + + +class LocationConfigRequest(BaseModel): + location_name: str + pages_to_scrape: int = 2 + adults: int = 2 + + +class HotelTierUpdate(BaseModel): + tier: str # 'own', 'competitor', 'market' + display_order: Optional[int] = None + + +class HotelResponse(BaseModel): + id: int + booking_com_id: str + name: str + booking_com_url: Optional[str] + star_rating: Optional[float] + review_score: Optional[float] + review_count: Optional[int] + tier: str + display_order: int + notes: Optional[str] + first_seen_at: Optional[datetime] + last_seen_at: Optional[datetime] + + +class RateResponse(BaseModel): + rate_date: str + hotel_id: int + hotel_name: str + tier: str + star_rating: Optional[float] + review_score: Optional[float] + availability_status: str + rate_gross: Optional[float] + room_type: Optional[str] + breakfast_included: Optional[bool] + free_cancellation: Optional[bool] + no_prepayment: Optional[bool] + rooms_left: Optional[int] + scraped_at: Optional[datetime] + + +class ScraperStatusResponse(BaseModel): + enabled: bool + paused: bool + pause_until: Optional[str] + backend: str + location_configured: bool + location_name: Optional[str] + last_scrape: Optional[dict] + + +# ============================================ +# SCRAPER STATUS & CONFIGURATION +# ============================================ + +@router.get("/status", response_model=ScraperStatusResponse) +async def get_scraper_status( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Get current scraper status and configuration.""" + # Get config values + config_result = await db.execute( + text(""" + SELECT config_key, config_value FROM system_config + WHERE config_key IN ( + 'booking_scraper_enabled', + 'booking_scraper_paused', + 'booking_scraper_pause_until', + 'booking_scraper_backend' + ) + """) + ) + config = {row.config_key: row.config_value for row in config_result.fetchall()} + + # Get location config + location_result = await db.execute( + text("SELECT location_name FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1") + ) + location_row = location_result.fetchone() + + # Get last scrape info + last_scrape_result = await db.execute( + text(""" + SELECT batch_id, scrape_type, started_at, completed_at, status, + hotels_found, rates_scraped, error_message + FROM booking_scrape_log + ORDER BY started_at DESC + LIMIT 1 + """) + ) + last_scrape_row = last_scrape_result.fetchone() + last_scrape = None + if last_scrape_row: + last_scrape = { + 'batch_id': str(last_scrape_row.batch_id), + 'scrape_type': last_scrape_row.scrape_type, + 'started_at': last_scrape_row.started_at.isoformat() if last_scrape_row.started_at else None, + 'completed_at': last_scrape_row.completed_at.isoformat() if last_scrape_row.completed_at else None, + 'status': last_scrape_row.status, + 'hotels_found': last_scrape_row.hotels_found, + 'rates_scraped': last_scrape_row.rates_scraped, + 'error_message': last_scrape_row.error_message, + } + + return ScraperStatusResponse( + enabled=config.get('booking_scraper_enabled', 'false') == 'true', + paused=config.get('booking_scraper_paused', 'false') == 'true', + pause_until=config.get('booking_scraper_pause_until'), + backend=config.get('booking_scraper_backend', 'playwright_local'), + location_configured=location_row is not None, + location_name=location_row.location_name if location_row else None, + last_scrape=last_scrape + ) + + +@router.post("/config/location") +async def set_location_config( + config: LocationConfigRequest, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Set the location to scrape for competitor rates.""" + # Deactivate existing configs + await db.execute( + text("UPDATE booking_scrape_config SET is_active = FALSE") + ) + + # Insert new config + await db.execute( + text(""" + INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, is_active) + VALUES (:location, :pages, :adults, TRUE) + """), + {'location': config.location_name, 'pages': config.pages_to_scrape, 'adults': config.adults} + ) + await db.commit() + + return {"status": "success", "location": config.location_name} + + +@router.post("/config/enable") +async def enable_scraper( + enabled: bool = True, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Enable or disable the booking.com scraper.""" + await db.execute( + text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_enabled'"), + {'val': 'true' if enabled else 'false'} + ) + await db.commit() + return {"status": "success", "enabled": enabled} + + +@router.post("/config/unpause") +async def unpause_scraper( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Manually unpause the scraper (clears blocking pause).""" + await db.execute( + text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'") + ) + await db.commit() + return {"status": "success", "message": "Scraper unpaused"} + + +# ============================================ +# MANUAL SCRAPE TRIGGER +# ============================================ + +def run_scrape_sync(from_date: date, to_date: date): + """Run scrape in sync context for background task.""" + import asyncio + from services.booking_scraper import run_manual_scrape, cleanup_stale_batches + + db = SyncSessionLocal() + try: + # Clean up any stale batches before starting + cleanup_stale_batches(db, max_age_minutes=60) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete(run_manual_scrape(db, from_date, to_date)) + logger.info(f"Background scrape completed: {result}") + finally: + loop.close() + except Exception as e: + logger.error(f"Background scrape failed: {e}", exc_info=True) + finally: + db.close() + + +@router.post("/scrape") +async def trigger_manual_scrape( + request: ScrapeRequest, + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Trigger a manual scrape for the specified date range. + + Runs in background - check /status for progress. + """ + try: + from_date = date.fromisoformat(request.from_date) + to_date = date.fromisoformat(request.to_date) if request.to_date else from_date + except ValueError as e: + raise HTTPException(status_code=400, detail=f"Invalid date format: {e}") + + if to_date < from_date: + raise HTTPException(status_code=400, detail="to_date must be after from_date") + + if (to_date - from_date).days > 30: + raise HTTPException(status_code=400, detail="Date range cannot exceed 30 days for manual scrape") + + # Check if location is configured + location_result = await db.execute( + text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1") + ) + if not location_result.fetchone(): + raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.") + + # Check if paused + paused_result = await db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'") + ) + paused_row = paused_result.fetchone() + if paused_row and paused_row.config_value == 'true': + raise HTTPException(status_code=400, detail="Scraper is currently paused. Use /unpause first or wait for cooldown.") + + # Start background task + background_tasks.add_task(run_scrape_sync, from_date, to_date) + + return { + "status": "started", + "from_date": from_date.isoformat(), + "to_date": to_date.isoformat(), + "message": "Scrape started in background. Check /status for progress." + } + + +# ============================================ +# HOTELS MANAGEMENT +# ============================================ + +@router.get("/hotels", response_model=List[HotelResponse]) +async def list_hotels( + tier: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + List all discovered hotels. + + Filter by tier: 'own', 'competitor', 'market', or None for all. + """ + query = """ + SELECT id, booking_com_id, name, booking_com_url, + star_rating, review_score, review_count, + tier, display_order, notes, first_seen_at, last_seen_at + FROM booking_com_hotels + WHERE is_active = TRUE + """ + params = {} + + if tier: + if tier not in ('own', 'competitor', 'market'): + raise HTTPException(status_code=400, detail="Invalid tier. Must be 'own', 'competitor', or 'market'") + query += " AND tier = :tier" + params['tier'] = tier + + query += " ORDER BY display_order, name" + + result = await db.execute(text(query), params) + + return [ + HotelResponse( + id=row.id, + booking_com_id=row.booking_com_id or '', + name=row.name, + booking_com_url=row.booking_com_url, + star_rating=float(row.star_rating) if row.star_rating else None, + review_score=float(row.review_score) if row.review_score else None, + review_count=row.review_count, + tier=row.tier, + display_order=row.display_order, + notes=row.notes, + first_seen_at=row.first_seen_at, + last_seen_at=row.last_seen_at + ) + for row in result.fetchall() + ] + + +@router.put("/hotels/{hotel_id}/tier") +async def update_hotel_tier( + hotel_id: int, + update: HotelTierUpdate, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Update a hotel's tier and display order. + + Tiers: + - 'own': Your hotel (for parity checking) + - 'competitor': Main competitors (full tracking) + - 'market': Other hotels (context only) + """ + if update.tier not in ('own', 'competitor', 'market'): + raise HTTPException(status_code=400, detail="Invalid tier") + + # If setting as 'own', clear any existing 'own' hotel + if update.tier == 'own': + await db.execute( + text("UPDATE booking_com_hotels SET tier = 'market' WHERE tier = 'own'") + ) + + # Update the hotel + set_clause = "tier = :tier" + params = {'hotel_id': hotel_id, 'tier': update.tier} + + if update.display_order is not None: + set_clause += ", display_order = :order" + params['order'] = update.display_order + + result = await db.execute( + text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id RETURNING id"), + params + ) + + if not result.fetchone(): + raise HTTPException(status_code=404, detail="Hotel not found") + + await db.commit() + + # If this is now the own hotel, update system config + if update.tier == 'own': + await db.execute( + text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_own_hotel_id'"), + {'val': str(hotel_id)} + ) + await db.commit() + + return {"status": "success", "hotel_id": hotel_id, "tier": update.tier} + + +@router.put("/hotels/{hotel_id}/notes") +async def update_hotel_notes( + hotel_id: int, + notes: str, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Update notes for a hotel.""" + result = await db.execute( + text("UPDATE booking_com_hotels SET notes = :notes WHERE id = :hotel_id RETURNING id"), + {'hotel_id': hotel_id, 'notes': notes} + ) + + if not result.fetchone(): + raise HTTPException(status_code=404, detail="Hotel not found") + + await db.commit() + return {"status": "success"} + + +# ============================================ +# COMPETITOR RATES MATRIX +# ============================================ + +@router.get("/matrix") +async def get_competitor_matrix( + from_date: Optional[str] = None, + to_date: Optional[str] = None, + include_market: bool = False, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get rate comparison matrix for competitors. + + Returns rates for own hotel and competitors, organized by date. + Set include_market=true to also include market tier hotels. + """ + today = date.today() + start = date.fromisoformat(from_date) if from_date else today + end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30) + + if end < start: + raise HTTPException(status_code=400, detail="to_date must be after from_date") + if (end - start).days > 90: + raise HTTPException(status_code=400, detail="Date range cannot exceed 90 days") + + tier_filter = "h.tier IN ('own', 'competitor')" + if include_market: + tier_filter = "h.tier IN ('own', 'competitor', 'market')" + + # Get hotels + hotels_result = await db.execute( + text(f""" + SELECT id, name, tier, display_order, star_rating, review_score, booking_com_url + FROM booking_com_hotels + WHERE is_active = TRUE AND {tier_filter.replace('h.', '')} + ORDER BY display_order, name + """) + ) + hotels = [dict(row._mapping) for row in hotels_result.fetchall()] + + # Get latest rates using the view + rates_result = await db.execute( + text(f""" + SELECT DISTINCT ON (r.hotel_id, r.rate_date) + r.hotel_id, + r.rate_date, + r.availability_status, + r.rate_gross, + r.room_type, + r.breakfast_included, + r.free_cancellation, + r.no_prepayment, + r.rooms_left, + r.scraped_at + FROM booking_com_rates r + JOIN booking_com_hotels h ON r.hotel_id = h.id + WHERE {tier_filter} + AND h.is_active = TRUE + AND r.rate_date >= :from_date AND r.rate_date <= :to_date + ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC + """), + {'from_date': start, 'to_date': end} + ) + + # Build matrix: hotel_id -> date -> rate data + rates_by_hotel: Dict[int, Dict[str, dict]] = {} + for row in rates_result.fetchall(): + hotel_id = row.hotel_id + rate_date = row.rate_date.isoformat() + + if hotel_id not in rates_by_hotel: + rates_by_hotel[hotel_id] = {} + + rates_by_hotel[hotel_id][rate_date] = { + 'availability_status': row.availability_status, + 'rate_gross': float(row.rate_gross) if row.rate_gross else None, + 'room_type': row.room_type, + 'breakfast_included': row.breakfast_included, + 'free_cancellation': row.free_cancellation, + 'no_prepayment': row.no_prepayment, + 'rooms_left': row.rooms_left, + 'scraped_at': row.scraped_at.isoformat() if row.scraped_at else None, + } + + # Build date list + dates = [] + current = start + while current <= end: + dates.append(current.isoformat()) + current += timedelta(days=1) + + return { + 'from_date': start.isoformat(), + 'to_date': end.isoformat(), + 'dates': dates, + 'hotels': hotels, + 'rates': rates_by_hotel + } + + +# ============================================ +# RATE PARITY (OWN HOTEL VS NEWBOOK) +# ============================================ + +@router.get("/parity") +async def get_rate_parity( + from_date: Optional[str] = None, + to_date: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get rate parity comparison between booking.com and Newbook rates. + + Compares scraped booking.com rates for own hotel against Newbook current rates. + """ + today = date.today() + start = date.fromisoformat(from_date) if from_date else today + end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30) + + # Get own hotel's booking.com rates + booking_rates_result = await db.execute( + text(""" + SELECT DISTINCT ON (r.rate_date) + r.rate_date, + r.rate_gross as booking_rate, + r.availability_status, + r.room_type as booking_room_type, + r.scraped_at + FROM booking_com_rates r + JOIN booking_com_hotels h ON r.hotel_id = h.id + WHERE h.tier = 'own' + AND r.rate_date >= :from_date AND r.rate_date <= :to_date + ORDER BY r.rate_date, r.scraped_at DESC + """), + {'from_date': start, 'to_date': end} + ) + booking_rates = {row.rate_date: dict(row._mapping) for row in booking_rates_result.fetchall()} + + # Get Newbook rates (best rate per date across categories) + newbook_rates_result = await db.execute( + text(""" + SELECT DISTINCT ON (rate_date) + rate_date, + rate_gross as newbook_rate, + category_id + FROM newbook_current_rates + WHERE rate_date >= :from_date AND rate_date <= :to_date + ORDER BY rate_date, valid_from DESC + """), + {'from_date': start, 'to_date': end} + ) + newbook_rates = {row.rate_date: dict(row._mapping) for row in newbook_rates_result.fetchall()} + + # Compare rates + parity_issues = [] + all_dates = set(booking_rates.keys()) | set(newbook_rates.keys()) + + for rate_date in sorted(all_dates): + booking = booking_rates.get(rate_date) + newbook = newbook_rates.get(rate_date) + + if not booking or not newbook: + continue + + booking_rate = booking.get('booking_rate') + newbook_rate = newbook.get('newbook_rate') + + if not booking_rate or not newbook_rate: + continue + + diff_pct = ((float(booking_rate) - float(newbook_rate)) / float(newbook_rate)) * 100 + + if abs(diff_pct) > 1: # More than 1% difference + parity_issues.append({ + 'rate_date': rate_date.isoformat(), + 'booking_rate': float(booking_rate), + 'newbook_rate': float(newbook_rate), + 'difference_pct': round(diff_pct, 2), + 'alert_type': 'higher' if diff_pct > 0 else 'lower', + 'booking_room_type': booking.get('booking_room_type'), + 'availability_status': booking.get('availability_status'), + }) + + return { + 'from_date': start.isoformat(), + 'to_date': end.isoformat(), + 'issues_count': len(parity_issues), + 'issues': parity_issues + } + + +# ============================================ +# PARITY ALERTS +# ============================================ + +@router.get("/parity/alerts") +async def get_parity_alerts( + status: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Get rate parity alerts.""" + query = """ + SELECT id, rate_date, room_category, newbook_rate, booking_com_rate, + difference_pct, alert_type, alert_status, created_at, + acknowledged_at, acknowledged_by, notes + FROM rate_parity_alerts + """ + params = {} + + if status: + query += " WHERE alert_status = :status" + params['status'] = status + + query += " ORDER BY rate_date DESC, created_at DESC LIMIT 100" + + result = await db.execute(text(query), params) + + return [ + { + 'id': row.id, + 'rate_date': row.rate_date.isoformat(), + 'room_category': row.room_category, + 'newbook_rate': float(row.newbook_rate) if row.newbook_rate else None, + 'booking_com_rate': float(row.booking_com_rate) if row.booking_com_rate else None, + 'difference_pct': float(row.difference_pct) if row.difference_pct else None, + 'alert_type': row.alert_type, + 'alert_status': row.alert_status, + 'created_at': row.created_at.isoformat() if row.created_at else None, + 'acknowledged_at': row.acknowledged_at.isoformat() if row.acknowledged_at else None, + 'acknowledged_by': row.acknowledged_by, + 'notes': row.notes, + } + for row in result.fetchall() + ] + + +@router.put("/parity/alerts/{alert_id}/acknowledge") +async def acknowledge_parity_alert( + alert_id: int, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Acknowledge a parity alert.""" + result = await db.execute( + text(""" + UPDATE rate_parity_alerts + SET alert_status = 'acknowledged', + acknowledged_at = NOW(), + acknowledged_by = :username + WHERE id = :alert_id + RETURNING id + """), + {'alert_id': alert_id, 'username': current_user.get('username', 'unknown')} + ) + + if not result.fetchone(): + raise HTTPException(status_code=404, detail="Alert not found") + + await db.commit() + return {"status": "success"} + + +# ============================================ +# QUEUE STATUS +# ============================================ + +@router.get("/queue-status") +async def get_queue_status( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Get current scrape queue status.""" + result = await db.execute( + text(""" + SELECT + status, + COUNT(*) as count, + MIN(rate_date) as earliest_date, + MAX(rate_date) as latest_date + FROM booking_scrape_queue + GROUP BY status + """) + ) + status_counts = {row.status: { + 'count': row.count, + 'earliest': row.earliest_date.isoformat() if row.earliest_date else None, + 'latest': row.latest_date.isoformat() if row.latest_date else None, + } for row in result.fetchall()} + + # Get retry items (failed but under max_attempts) + retry_result = await db.execute( + text(""" + SELECT COUNT(*) as count + FROM booking_scrape_queue + WHERE status = 'pending' AND attempts > 0 + """) + ) + retry_count = retry_result.fetchone().count + + return { + 'statuses': status_counts, + 'retries_pending': retry_count, + 'total_pending': status_counts.get('pending', {}).get('count', 0), + 'total_completed': status_counts.get('completed', {}).get('count', 0), + 'total_failed': status_counts.get('failed', {}).get('count', 0), + } + + +# ============================================ +# SCHEDULE INFO +# ============================================ + +@router.get("/schedule-info") +async def get_schedule_info( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Get information about the scraping schedule.""" + # Get configured time + time_result = await db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_daily_time'") + ) + time_row = time_result.fetchone() + daily_time = time_row.config_value if time_row and time_row.config_value else '05:30' + + # Calculate what today's schedule would look like + from jobs.scrape_booking_rates import get_high_priority_dates, get_medium_priority_dates, get_low_priority_dates + high = get_high_priority_dates() + medium = get_medium_priority_dates() + low = get_low_priority_dates() + + today = date.today() + weekday_name = today.strftime('%A') + + return { + 'daily_time': daily_time, + 'today': today.isoformat(), + 'weekday': weekday_name, + 'tiers': { + 'high': { + 'description': 'Next 30 days (scraped first)', + 'dates_today': len(high), + 'range': f'{high[0].isoformat()} to {high[-1].isoformat()}' if high else None, + }, + 'medium': { + 'description': 'Days 31-180 (scraped after high priority)', + 'dates_today': len(medium), + 'range': f'{medium[0].isoformat()} to {medium[-1].isoformat()}' if medium else None, + }, + 'low': { + 'description': 'Days 181-365 (scraped last, or until rate limit)', + 'dates_today': len(low), + 'range': f'{low[0].isoformat()} to {low[-1].isoformat()}' if low else None, + }, + }, + 'total_dates_today': len(set(high + medium + low)), + } + + +# ============================================ +# SCRAPE COVERAGE (365-day view) +# ============================================ + +@router.get("/scrape-coverage") +async def get_scrape_coverage( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Get 365-day scrape coverage showing last scraped time + and next expected scrape for every date. + """ + today = date.today() + end = today + timedelta(days=365) + + # Get latest scraped_at per date (across all hotels) + result = await db.execute( + text(""" + SELECT rate_date, MAX(scraped_at) as last_scraped + FROM booking_com_rates + WHERE rate_date >= :from_date AND rate_date <= :to_date + GROUP BY rate_date + """), + {'from_date': today, 'to_date': end} + ) + scraped_map = {row.rate_date: row.last_scraped for row in result.fetchall()} + + # Compute tier and next scrape for each date + from jobs.scrape_booking_rates import compute_next_scrape_for_date + + coverage = [] + for offset in range(366): + d = today + timedelta(days=offset) + tier, next_scrape = compute_next_scrape_for_date(d) + last_scraped = scraped_map.get(d) + + coverage.append({ + 'date': d.isoformat(), + 'tier': tier, + 'last_scraped': last_scraped.isoformat() if last_scraped else None, + 'next_expected': next_scrape.isoformat() if next_scrape else None, + }) + + return { + 'today': today.isoformat(), + 'coverage': coverage, + } + + +# ============================================ +# BOOKING.COM AVAILABILITY CHECK (for Bookability page) +# ============================================ + +@router.get("/booking-availability") +async def get_booking_availability( + from_date: Optional[str] = None, + to_date: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """ + Check own hotel's availability on booking.com. + + Returns a simple summary: for each date in the range, whether the own hotel + appears available on booking.com based on the latest scrape data. + """ + today = date.today() + start = date.fromisoformat(from_date) if from_date else today + end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30) + + # Get own hotel's latest scraped availability + result = await db.execute( + text(""" + SELECT DISTINCT ON (r.rate_date) + r.rate_date, + r.availability_status, + r.rate_gross, + r.scraped_at + FROM booking_com_rates r + JOIN booking_com_hotels h ON r.hotel_id = h.id + WHERE h.tier = 'own' + AND r.rate_date >= :from_date AND r.rate_date <= :to_date + ORDER BY r.rate_date, r.scraped_at DESC + """), + {'from_date': start, 'to_date': end} + ) + rows = result.fetchall() + + if not rows: + return { + 'has_own_hotel': False, + 'dates_checked': 0, + 'dates_available': 0, + 'dates_sold_out': 0, + 'dates_no_data': 0, + 'latest_scrape': None, + 'dates': {}, + } + + dates_map = {} + dates_available = 0 + dates_sold_out = 0 + dates_no_data = 0 + latest_scrape = None + + for row in rows: + status = row.availability_status + dates_map[row.rate_date.isoformat()] = { + 'status': status, + 'rate': float(row.rate_gross) if row.rate_gross else None, + } + if status == 'available': + dates_available += 1 + elif status == 'sold_out': + dates_sold_out += 1 + else: + dates_no_data += 1 + + if row.scraped_at and (not latest_scrape or row.scraped_at > latest_scrape): + latest_scrape = row.scraped_at + + return { + 'has_own_hotel': True, + 'dates_checked': len(rows), + 'dates_available': dates_available, + 'dates_sold_out': dates_sold_out, + 'dates_no_data': dates_no_data, + 'latest_scrape': latest_scrape.isoformat() if latest_scrape else None, + 'dates': dates_map, + } + + +# ============================================ +# SCRAPE HISTORY +# ============================================ + +@router.get("/scrape-history") +async def get_scrape_history( + limit: int = 20, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Get recent scrape batch history.""" + result = await db.execute( + text(""" + SELECT batch_id, scrape_type, started_at, completed_at, status, + dates_queued, dates_completed, dates_failed, + hotels_found, rates_scraped, error_message, + blocked_at, resume_after + FROM booking_scrape_log + ORDER BY started_at DESC + LIMIT :limit + """), + {'limit': limit} + ) + + return [ + { + 'batch_id': str(row.batch_id), + 'scrape_type': row.scrape_type, + 'started_at': row.started_at.isoformat() if row.started_at else None, + 'completed_at': row.completed_at.isoformat() if row.completed_at else None, + 'status': row.status, + 'dates_queued': row.dates_queued, + 'dates_completed': row.dates_completed, + 'dates_failed': row.dates_failed, + 'hotels_found': row.hotels_found, + 'rates_scraped': row.rates_scraped, + 'error_message': row.error_message, + 'blocked_at': row.blocked_at.isoformat() if row.blocked_at else None, + 'resume_after': row.resume_after.isoformat() if row.resume_after else None, + } + for row in result.fetchall() + ] diff --git a/backend/api/direct.py b/backend/api/direct.py new file mode 100644 index 0000000..06802db --- /dev/null +++ b/backend/api/direct.py @@ -0,0 +1,337 @@ +""" +Direct booking engine API — hotel management, discovery, scrape control, rate data. +""" +import asyncio +import json +import logging +from datetime import date, timedelta +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks +from pydantic import BaseModel +from sqlalchemy import text + +from database import AsyncSessionLocal, SyncSessionLocal +from auth import get_current_user, require_cap +from services.direct_profiles import PROFILES, detect_profile, get_profile +from services.direct_scraper import run_discovery, get_discovery_status, run_scrape + +router = APIRouter() +log = logging.getLogger(__name__) + + +# ─── Pydantic models ───────────────────────────────────────────────────────── + +class HotelCreate(BaseModel): + name: str + profile_name: str + params: dict + + +class HotelUpdate(BaseModel): + name: Optional[str] = None + room_labels: Optional[dict] = None + rate_labels: Optional[dict] = None + room_order: Optional[list] = None + benchmark_room: Optional[str] = None + benchmark_rate: Optional[str] = None + tier_base_room: Optional[str] = None + tier_offsets: Optional[dict] = None + scrape_enabled: Optional[bool] = None + params: Optional[dict] = None + + +class DetectRequest(BaseModel): + url: str + + +# ─── Profiles ──────────────────────────────────────────────────────────────── + +@router.get("/profiles") +async def list_profiles(user=Depends(get_current_user)): + require_cap(user, "view_direct_rates") + return [ + { + "name": name, + "label": cls.label, + "required_params": cls.required_params, + } + for name, cls in PROFILES.items() + ] + + +@router.post("/profiles/detect") +async def detect_engine(req: DetectRequest, user=Depends(get_current_user)): + require_cap(user, "manage_hotels") + result = detect_profile(req.url) + if not result: + raise HTTPException(status_code=422, detail="Could not detect booking engine from URL") + return result + + +# ─── Hotel CRUD ─────────────────────────────────────────────────────────────── + +@router.get("/hotels") +async def list_hotels(user=Depends(get_current_user)): + require_cap(user, "view_direct_rates") + async with AsyncSessionLocal() as db: + result = await db.execute( + text("""SELECT h.id, h.name, h.profile_name, h.params, + h.room_labels, h.rate_labels, h.room_order, + h.benchmark_room, h.benchmark_rate, h.tier_base_room, h.tier_offsets, + h.scrape_enabled, h.last_scraped_at, + COUNT(DISTINCT r.stay_date) AS scraped_dates, + MAX(r.scraped_at) AS last_rate_at + FROM direct_competitor_hotels h + LEFT JOIN direct_rates r ON r.hotel_id = h.id + GROUP BY h.id + ORDER BY h.id""") + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + +@router.post("/hotels", status_code=201) +async def create_hotel(body: HotelCreate, user=Depends(get_current_user)): + require_cap(user, "manage_hotels") + if body.profile_name not in PROFILES: + raise HTTPException(status_code=400, detail=f"Unknown profile: {body.profile_name}") + async with AsyncSessionLocal() as db: + result = await db.execute( + text("""INSERT INTO direct_competitor_hotels (name, profile_name, params) + VALUES (:name, :profile, :params) RETURNING id"""), + {"name": body.name, "profile": body.profile_name, "params": json.dumps(body.params)} + ) + new_id = result.fetchone()[0] + await db.commit() + return {"id": new_id, "name": body.name, "profile_name": body.profile_name} + + +@router.put("/hotels/{hotel_id}") +async def update_hotel(hotel_id: int, body: HotelUpdate, user=Depends(get_current_user)): + require_cap(user, "manage_hotels") + updates = {} + if body.name is not None: updates["name"] = body.name + if body.scrape_enabled is not None: updates["scrape_enabled"] = body.scrape_enabled + if body.benchmark_room is not None: updates["benchmark_room"] = body.benchmark_room + if body.benchmark_rate is not None: updates["benchmark_rate"] = body.benchmark_rate + if body.tier_base_room is not None: updates["tier_base_room"] = body.tier_base_room + if body.params is not None: updates["params"] = json.dumps(body.params) + if body.room_labels is not None: updates["room_labels"] = json.dumps(body.room_labels) + if body.rate_labels is not None: updates["rate_labels"] = json.dumps(body.rate_labels) + if body.room_order is not None: updates["room_order"] = json.dumps(body.room_order) + if body.tier_offsets is not None: updates["tier_offsets"] = json.dumps(body.tier_offsets) + + if not updates: + raise HTTPException(status_code=400, detail="No fields to update") + + set_clause = ", ".join(f"{k} = :{k}" for k in updates) + updates["hotel_id"] = hotel_id + async with AsyncSessionLocal() as db: + await db.execute( + text(f"UPDATE direct_competitor_hotels SET {set_clause} WHERE id = :hotel_id"), + updates + ) + await db.commit() + return {"ok": True} + + +@router.delete("/hotels/{hotel_id}", status_code=204) +async def delete_hotel(hotel_id: int, user=Depends(get_current_user)): + require_cap(user, "manage_hotels") + async with AsyncSessionLocal() as db: + await db.execute( + text("DELETE FROM direct_competitor_hotels WHERE id = :id"), + {"id": hotel_id} + ) + await db.commit() + + +# ─── Discovery ─────────────────────────────────────────────────────────────── + +@router.post("/hotels/{hotel_id}/discover") +async def trigger_discovery(hotel_id: int, background_tasks: BackgroundTasks, user=Depends(get_current_user)): + require_cap(user, "manage_hotels") + async with AsyncSessionLocal() as db: + row = await db.execute( + text("SELECT profile_name, params FROM direct_competitor_hotels WHERE id = :id"), + {"id": hotel_id} + ) + hotel = row.mappings().fetchone() + if not hotel: + raise HTTPException(status_code=404, detail="Hotel not found") + + background_tasks.add_task( + asyncio.get_event_loop().run_until_complete, + run_discovery(hotel_id, hotel["profile_name"], hotel["params"]) + ) + return {"status": "discovery started", "hotel_id": hotel_id} + + +@router.get("/hotels/{hotel_id}/discovery-status") +async def discovery_status(hotel_id: int, user=Depends(get_current_user)): + require_cap(user, "view_direct_rates") + return get_discovery_status(hotel_id) + + +# ─── Manual scrape trigger ──────────────────────────────────────────────────── + +@router.post("/hotels/{hotel_id}/scrape") +async def trigger_scrape(hotel_id: int, background_tasks: BackgroundTasks, user=Depends(get_current_user)): + require_cap(user, "manage_scraper") + async with AsyncSessionLocal() as db: + row = await db.execute( + text("SELECT profile_name, params FROM direct_competitor_hotels WHERE id = :id"), + {"id": hotel_id} + ) + hotel = row.mappings().fetchone() + if not hotel: + raise HTTPException(status_code=404, detail="Hotel not found") + + def _bg(): + run_scrape(hotel_id, hotel["profile_name"], hotel["params"]) + + background_tasks.add_task(_bg) + return {"status": "scrape started", "hotel_id": hotel_id} + + +# ─── Rate data ──────────────────────────────────────────────────────────────── + +@router.get("/hotels/{hotel_id}/dates") +async def hotel_dates( + hotel_id: int, + from_date: date = None, + to_date: date = None, + user=Depends(get_current_user) +): + require_cap(user, "view_direct_rates") + if from_date is None: + from_date = date.today() + if to_date is None: + to_date = date.today() + timedelta(days=89) + + async with AsyncSessionLocal() as db: + # Hotel config for labels/tier offsets + cfg_row = await db.execute( + text("SELECT name, room_labels, rate_labels, tier_offsets, benchmark_room, benchmark_rate, tier_base_room FROM direct_competitor_hotels WHERE id = :id"), + {"id": hotel_id} + ) + hotel = cfg_row.mappings().fetchone() + if not hotel: + raise HTTPException(status_code=404, detail="Hotel not found") + + # Latest snapshot per date: cheapest available price_incl + rates_result = await db.execute( + text(""" + SELECT + r.stay_date, + MIN(r.price_incl) FILTER (WHERE r.availability > 0) AS cheapest_rate, + BOOL_OR(r.availability > 0) AS has_availability, + BOOL_OR(r.min_stay_nights IS NOT NULL + AND r.min_stay_nights > 1) AS has_min_stay, + MAX(r.scraped_at) AS scraped_at + FROM ( + SELECT DISTINCT ON (room_id, rate_id) + stay_date, room_id, rate_id, availability, + price_incl, min_stay_nights, scraped_at + FROM direct_rates + WHERE hotel_id = :hid + AND stay_date BETWEEN :fd AND :td + ORDER BY room_id, rate_id, scraped_at DESC + ) r + GROUP BY r.stay_date + ORDER BY r.stay_date + """), + {"hid": hotel_id, "fd": from_date, "td": to_date} + ) + dates = [dict(r) for r in rates_result.mappings().all()] + + return { + "hotel_id": hotel_id, + "hotel_name": hotel["name"], + "from_date": str(from_date), + "to_date": str(to_date), + "dates": dates, + } + + +@router.get("/hotels/{hotel_id}/date/{rate_date}/rooms") +async def hotel_date_rooms( + hotel_id: int, + rate_date: date, + user=Depends(get_current_user) +): + require_cap(user, "view_direct_rates") + async with AsyncSessionLocal() as db: + cfg_row = await db.execute( + text("SELECT room_labels, rate_labels, tier_offsets, tier_base_room, benchmark_room, benchmark_rate FROM direct_competitor_hotels WHERE id = :id"), + {"id": hotel_id} + ) + hotel = cfg_row.mappings().fetchone() + if not hotel: + raise HTTPException(status_code=404, detail="Hotel not found") + + # Latest snapshot per room/rate for this date + rooms_result = await db.execute( + text(""" + SELECT DISTINCT ON (room_id, rate_id) + room_id, rate_id, availability, price_excl, price_incl, + currency, min_stay_nights, scraped_at + FROM direct_rates + WHERE hotel_id = :hid AND stay_date = :sd + ORDER BY room_id, rate_id, scraped_at DESC + """), + {"hid": hotel_id, "sd": rate_date} + ) + rooms = [dict(r) for r in rooms_result.mappings().all()] + + # Resolve tier-normalised benchmark rates + room_labels = hotel["room_labels"] or {} + rate_labels = hotel["rate_labels"] or {} + tier_offsets = hotel["tier_offsets"] or {} + tier_base_room = hotel["tier_base_room"] + benchmark_room = hotel["benchmark_room"] + benchmark_rate = hotel["benchmark_rate"] + + # Find benchmark price + bench_price = None + if benchmark_room and benchmark_rate: + bench_match = next( + (r for r in rooms + if r["room_id"] == benchmark_room and r["rate_id"] == benchmark_rate + and r["availability"] > 0 and r["price_incl"]), + None + ) + if bench_match: + bench_price = float(bench_match["price_incl"]) + elif tier_base_room and tier_offsets: + base_match = next( + (r for r in rooms + if r["room_id"] == tier_base_room and r["availability"] > 0 and r["price_incl"]), + None + ) + if base_match: + base_price = float(base_match["price_incl"]) + bench_offset = tier_offsets.get(benchmark_room, 0) + base_offset = tier_offsets.get(tier_base_room, 0) + bench_price = base_price - base_offset + bench_offset + + enriched = [] + for r in rooms: + r["room_label"] = room_labels.get(r["room_id"], r["room_id"]) + r["rate_label"] = rate_labels.get(r["rate_id"], r["rate_id"]) + # Derive bench_rate for this room from tier offsets + r["bench_rate"] = None + if bench_price is not None and tier_offsets and tier_base_room: + room_offset = tier_offsets.get(r["room_id"]) + bench_offset = tier_offsets.get(benchmark_room, 0) + if room_offset is not None: + r["bench_rate"] = round(bench_price + (room_offset - bench_offset), 2) + enriched.append(r) + + return { + "hotel_id": hotel_id, + "date": str(rate_date), + "bench_price": bench_price, + "rooms": enriched, + } diff --git a/backend/auth.py b/backend/auth.py new file mode 100644 index 0000000..ca1030a --- /dev/null +++ b/backend/auth.py @@ -0,0 +1,69 @@ +""" +Auth middleware — verifies the stack's hnf_session cookie using the shared +CENTRAL_AUTH_SECRET. +""" +import os +from typing import Optional + +from fastapi import Depends, HTTPException, Request, status +from jose import JWTError, jwt + +CENTRAL_AUTH_SECRET = os.getenv("CENTRAL_AUTH_SECRET", "") +JWT_ALGORITHM = "HS256" +APP_SLUG = os.getenv("APP_SLUG", "rates") + + +async def get_current_user(request: Request) -> dict: + token = request.cookies.get("hnf_session") + if not token: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + + try: + payload = jwt.decode(token, CENTRAL_AUTH_SECRET, algorithms=[JWT_ALGORITHM]) + except JWTError: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session") + + apps = payload.get("apps", []) + if APP_SLUG not in apps: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No permission for this app") + + prefix = f"{APP_SLUG}:" + raw_caps = payload.get("caps", []) + if isinstance(raw_caps, list): + caps = [c[len(prefix):] for c in raw_caps if c.startswith(prefix)] + else: + caps = [] + + is_admin = payload.get("is_admin", False) + + return { + "id": 0, + "username": payload.get("sub", ""), + "email": payload.get("sub", ""), + "display_name": payload.get("name", ""), + "name": payload.get("name", ""), + "is_admin": is_admin, + "caps": caps, + "role": "admin" if is_admin else "user", + } + + +def has_cap(user: dict, cap: str) -> bool: + return user.get("is_admin", False) or cap in user.get("caps", []) + + +def require_cap(cap: str): + async def checker(user: dict = Depends(get_current_user)): + if not has_cap(user, cap): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Missing capability: {cap}", + ) + return user + return checker + + +async def get_admin_user(user: dict = Depends(get_current_user)) -> dict: + if not user.get("is_admin", False): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + return user diff --git a/backend/database.py b/backend/database.py new file mode 100644 index 0000000..e131386 --- /dev/null +++ b/backend/database.py @@ -0,0 +1,37 @@ +""" +Database connection and session management +""" +import os +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker, declarative_base +from sqlalchemy import create_engine + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://rates:rates_secret@localhost:5432/rates_db") + +ASYNC_DATABASE_URL = DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://") + +async_engine = create_async_engine(ASYNC_DATABASE_URL, echo=False) +AsyncSessionLocal = sessionmaker( + async_engine, class_=AsyncSession, expire_on_commit=False +) + +sync_engine = create_engine(DATABASE_URL) +SyncSessionLocal = sessionmaker(bind=sync_engine) + +Base = declarative_base() + + +async def get_db(): + async with AsyncSessionLocal() as session: + try: + yield session + finally: + await session.close() + + +def get_sync_db(): + db = SyncSessionLocal() + try: + yield db + finally: + db.close() diff --git a/backend/jobs/__init__.py b/backend/jobs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/jobs/fetch_current_rates.py b/backend/jobs/fetch_current_rates.py new file mode 100644 index 0000000..8c7d5ff --- /dev/null +++ b/backend/jobs/fetch_current_rates.py @@ -0,0 +1,370 @@ +""" +Fetch Current Rates Job + +Fetches current rack rates from Newbook API and populates newbook_current_rates table. +These rates are used by pickup-v2 model for upper bound calculations in confidence shading. + +Uses a snapshot model - only inserts new rows when rates change, otherwise updates last_verified_at. +This allows tracking rate history over time. + +Schedule: Daily at 5:20 AM (before pace snapshot runs) + +Processing: Day-by-day with progressive DB commits. Each date is fully processed +(single-night fetch + inline multi-night verification) and saved before moving to the next. +If the job fails partway, all previously processed dates are preserved. +""" +import json +import logging +import re +from datetime import date, timedelta +from decimal import Decimal +from typing import Dict, Any, Optional, Set + +import asyncio +from sqlalchemy import text +from database import SyncSessionLocal + +logger = logging.getLogger(__name__) + +COMMIT_BATCH_SIZE = 10 # Commit to DB every N days + + +def rates_changed(old_rate: Optional[Dict], new_rate: Dict) -> bool: + """ + Compare old and new rates to determine if they've changed. + + Compares gross rate, net rate, and tariff availability status. + Returns True if rates have changed, False if they're the same. + """ + if old_rate is None: + return True # No existing rate, need to insert + + # Compare gross and net rates + old_gross = float(old_rate.get('rate_gross') or 0) + new_gross = float(new_rate.get('gross_rate') or 0) + if abs(old_gross - new_gross) > 0.01: + return True + + old_net = float(old_rate.get('rate_net') or 0) + new_net = float(new_rate.get('net_rate') or 0) + if abs(old_net - new_net) > 0.01: + return True + + # Compare tariff availability + old_tariffs = old_rate.get('tariffs_data', {}) + if isinstance(old_tariffs, str): + try: + old_tariffs = json.loads(old_tariffs) + except json.JSONDecodeError: + old_tariffs = {} + + new_tariffs = new_rate.get('tariffs_data', {}) + + old_tariff_list = old_tariffs.get('tariffs', []) + new_tariff_list = new_tariffs.get('tariffs', []) + + # Different number of tariffs + if len(old_tariff_list) != len(new_tariff_list): + return True + + # Compare each tariff's key attributes + for old_t, new_t in zip(old_tariff_list, new_tariff_list): + # Name changed + if old_t.get('name') != new_t.get('name'): + return True + # Availability status changed + if old_t.get('success') != new_t.get('success'): + return True + # Rate changed significantly + old_rate_val = float(old_t.get('rate') or 0) + new_rate_val = float(new_t.get('rate') or 0) + if abs(old_rate_val - new_rate_val) > 0.01: + return True + # Min stay changed + if old_t.get('min_stay') != new_t.get('min_stay'): + return True + # Multi-night availability changed + if old_t.get('available_for_min_stay') != new_t.get('available_for_min_stay'): + return True + + return False + + +def save_rate_snapshot(db, category_id: str, rate_date: date, rate: Dict) -> str: + """ + Save rate to database using snapshot logic. + + If rate has changed from latest version, insert new row. + If rate is the same, just update last_verified_at. + + Returns: 'inserted', 'verified', or 'error' + """ + gross_rate = rate.get('gross_rate') + net_rate = rate.get('net_rate') + tariffs_data = rate.get('tariffs_data', {}) + + # Get the latest rate for this category/date + existing = db.execute( + text(""" + SELECT id, rate_gross, rate_net, tariffs_data + FROM newbook_current_rates + WHERE category_id = :category_id AND rate_date = :rate_date + ORDER BY valid_from DESC + LIMIT 1 + """), + {"category_id": category_id, "rate_date": rate_date} + ).fetchone() + + if existing: + existing_dict = { + 'rate_gross': existing.rate_gross, + 'rate_net': existing.rate_net, + 'tariffs_data': existing.tariffs_data + } + else: + existing_dict = None + + if rates_changed(existing_dict, rate): + # Rates changed - insert new snapshot + db.execute( + text(""" + INSERT INTO newbook_current_rates + (category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from, last_verified_at) + VALUES (:category_id, :rate_date, :rate_gross, :rate_net, + CAST(:tariffs_data AS jsonb), NOW(), NOW()) + """), + { + "category_id": category_id, + "rate_date": rate_date, + "rate_gross": gross_rate, + "rate_net": net_rate, + "tariffs_data": json.dumps(tariffs_data) + } + ) + return 'inserted' + else: + # Rates unchanged - just verify + db.execute( + text(""" + UPDATE newbook_current_rates + SET last_verified_at = NOW() + WHERE id = :id + """), + {"id": existing.id} + ) + return 'verified' + + +def needs_multi_night_check(tariff: Dict, days_ahead: int) -> Optional[int]: + """ + Check if a tariff needs multi-night verification. + + Returns the min_stay value if a multi-night check is needed, None otherwise. + Skips tariffs with advance booking restrictions that aren't met. + """ + min_stay = tariff.get('min_stay') + if not min_stay or min_stay <= 1: + return None + if tariff.get('success', False): + return None # Already available as single-night, no recheck needed + + # Check for advance booking requirement + message = tariff.get('message', '') or '' + advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE) + if advance_match: + min_advance_days = int(advance_match.group(1)) + if days_ahead < min_advance_days: + return None # Within advance period - recheck won't help + + return min_stay + + +async def run_fetch_current_rates(horizon_days: int = 720, start_date: date = None): + """ + Fetch current rates for all included categories and store in database. + + Args: + horizon_days: Number of days ahead to fetch (default 720 for scheduled, configurable for manual) + start_date: Start date for fetch (default today) + + Processing: Day-by-day with progressive commits. + For each date: + 1. Fetch single-night rates (all categories in one API call) + 2. Check if any tariffs need multi-night verification + 3. If so, run multi-night check immediately for that date + 4. Save all rates for that date to DB + 5. Commit every COMMIT_BATCH_SIZE days + + This means if the job fails at day 400, the first 390+ days are already saved. + """ + logger.info(f"Starting current rates fetch ({horizon_days} days)") + + db = next(iter([SyncSessionLocal()])) + today = start_date or date.today() + + try: + # Get VAT rate from config + vat_result = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'") + ).fetchone() + vat_rate_str = vat_result.config_value if vat_result and vat_result.config_value else '0.20' + + # Get all included room categories + cat_result = db.execute( + text("SELECT site_id FROM newbook_room_categories WHERE is_included = true") + ) + included_categories = set(row.site_id for row in cat_result.fetchall()) + + if not included_categories: + logger.warning("No included room categories found") + return + + logger.info(f"Fetching rates for {len(included_categories)} categories") + + # Import rates client + import base64 + from services.newbook_rates_client import NewbookRatesClient + + # Credentials: central Settings service first, app-local config fallback + from services.central_settings import get_newbook_credentials_sync + creds = get_newbook_credentials_sync() + + if not creds: + config_result = db.execute( + text(""" + SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted + FROM system_config + WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region') + """) + ) + config = {} + for row in config_result.fetchall(): + value = row.config_value + if row.is_encrypted and value: + try: + value = base64.b64decode(value.encode()).decode() + except Exception: + pass # Use raw value if decryption fails + config[row.config_key] = value + + if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']): + logger.error("Newbook credentials not configured") + return + + creds = { + 'api_key': config['newbook_api_key'], + 'username': config['newbook_username'], + 'password': config['newbook_password'], + 'region': config['newbook_region'], + } + + # Create client + client = NewbookRatesClient( + api_key=creds['api_key'], + username=creds['username'], + password=creds['password'], + region=creds['region'], + vat_rate=Decimal(vat_rate_str) + ) + + async with client: + inserted_total = 0 + verified_total = 0 + multi_night_checks = 0 + skipped_advance = 0 + current_date = today + day_count = 0 + + while current_date <= today + timedelta(days=horizon_days): + day_count += 1 + days_ahead = (current_date - today).days + + try: + # Step 1: Fetch single-night rates for all categories on this date + day_rates = await client.fetch_single_date_all_categories( + current_date, guests_adults=2, guests_children=0 + ) + + # Step 2: Check for multi-night verification needs and run inline + # Collect unique min_stay values needed for this date + nights_needed: Set[int] = set() + for cat_id, rates in day_rates.items(): + if cat_id not in included_categories: + continue + for rate in rates: + for tariff in rate.get('tariffs_data', {}).get('tariffs', []): + check = needs_multi_night_check(tariff, days_ahead) + if check: + nights_needed.add(check) + elif tariff.get('min_stay') and tariff['min_stay'] > 1 and not tariff.get('success', False): + skipped_advance += 1 + + # Step 3: Run multi-night checks for this date if needed + multi_night_results: Dict[int, Dict[str, Dict[str, bool]]] = {} + for nights in sorted(nights_needed): + try: + result = await client.fetch_multi_night_for_date( + current_date, nights + ) + multi_night_results[nights] = result + multi_night_checks += 1 + await asyncio.sleep(1.0) # Rate limiting + except Exception as e: + logger.warning(f"Multi-night check failed for {current_date} ({nights}n): {e}") + + # Step 4: Update tariffs with multi-night results and save to DB + for cat_id, rates in day_rates.items(): + if cat_id not in included_categories: + continue + for rate in rates: + tariffs_data = rate.get('tariffs_data', {}) + # Apply multi-night results to tariffs + for tariff in tariffs_data.get('tariffs', []): + min_stay = tariff.get('min_stay') + if min_stay and min_stay > 1 and min_stay in multi_night_results: + cat_availability = multi_night_results[min_stay].get(cat_id, {}) + tariff_name = tariff.get('name', '') + tariff['available_for_min_stay'] = cat_availability.get(tariff_name, False) + + # Save to DB + result = save_rate_snapshot(db, cat_id, current_date, rate) + if result == 'inserted': + inserted_total += 1 + elif result == 'verified': + verified_total += 1 + + if day_count % 50 == 0 or nights_needed: + logger.info( + f"Day {day_count}/{horizon_days}: {current_date}" + f" | {inserted_total} new, {verified_total} verified" + f"{f' | {len(nights_needed)} multi-night checks' if nights_needed else ''}" + ) + + except Exception as e: + logger.warning(f"Failed to fetch rates for {current_date}: {e}") + + # Step 5: Commit periodically + if day_count % COMMIT_BATCH_SIZE == 0: + db.commit() + + current_date += timedelta(days=1) + await asyncio.sleep(1.0) # Rate limiting between days + + # Final commit for remaining days + db.commit() + + if skipped_advance > 0: + logger.info(f"Skipped {skipped_advance} multi-night checks (advance booking restriction)") + logger.info( + f"Complete: {inserted_total} new snapshots, {verified_total} verified unchanged, " + f"{multi_night_checks} multi-night checks" + ) + + logger.info("Current rates fetch completed successfully") + + except Exception as e: + logger.error(f"Current rates fetch failed: {e}") + db.rollback() + raise + finally: + db.close() diff --git a/backend/jobs/scrape_booking_rates.py b/backend/jobs/scrape_booking_rates.py new file mode 100644 index 0000000..e9371c5 --- /dev/null +++ b/backend/jobs/scrape_booking_rates.py @@ -0,0 +1,165 @@ +""" +Scheduled Booking.com Rate Scraping Job + +Priority-based scheduling for 365-day coverage (all queued daily): +- High (priority 10): next 30 days +- Medium (priority 5): days 31-180 +- Low (priority 2): days 181-365 + +Queue processes in priority order. If rate-limited/blocked, lower priority +dates remain queued for the next run. + +Uses a queue-based approach: +1. Populate the queue with dates and priorities +2. Process the queue in priority order +3. Failed dates are retried (up to 3 attempts) +4. On blocking, the queue pauses and resumes after cooldown + +Schedule: Daily at configurable time (default 05:30) +""" +import asyncio +import logging +from datetime import date, timedelta + +from sqlalchemy import text +from database import SyncSessionLocal +from services.booking_scraper import ( + populate_queue, + process_queue, + clear_old_queue_items, + cleanup_stale_batches, + get_scrape_config, +) + +logger = logging.getLogger(__name__) + +# Priority levels (higher = processed first) +PRIORITY_HIGH = 10 # 0-30 days +PRIORITY_MEDIUM = 5 # 31-180 days +PRIORITY_LOW = 2 # 181-365 days + + +def get_high_priority_dates() -> list[date]: + """High priority: today + 30 days.""" + today = date.today() + return [today + timedelta(days=i) for i in range(31)] + + +def get_medium_priority_dates() -> list[date]: + """Medium priority: days 31-180.""" + today = date.today() + return [today + timedelta(days=i) for i in range(31, 181)] + + +def get_low_priority_dates() -> list[date]: + """Low priority: days 181-365.""" + today = date.today() + return [today + timedelta(days=i) for i in range(181, 366)] + + +def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]: + """ + For a target date, determine its priority tier and when it will next be scraped. + + Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'. + All dates are queued daily, so next scrape is always today (or tomorrow if + today's run has passed). + """ + today = date.today() + offset = (target_date - today).days + + if offset < 0: + return ('none', None) + if offset > 365: + return ('none', None) + + # All tiers run daily - next scrape is today + if offset <= 30: + return ('high', today) + elif offset <= 180: + return ('medium', today) + else: + return ('low', today) + + +def run_scheduled_booking_scrape(): + """ + Main scheduled job: populate queue with today's dates, then process. + """ + db = SyncSessionLocal() + try: + # Check if scraper is enabled + result = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'") + ).fetchone() + if not result or result.config_value != 'true': + logger.debug("Scheduled booking scrape skipped (disabled)") + return + + if not get_scrape_config(db): + logger.warning("Scheduled booking scrape skipped (no location configured)") + return + + # Clean up stale running batches and old queue items + cleanup_stale_batches(db, max_age_minutes=120) + clear_old_queue_items(db, days=3) + + # Gather dates with priorities + high = get_high_priority_dates() + medium = get_medium_priority_dates() + low = get_low_priority_dates() + + priorities = {} + for d in high: + priorities[d] = PRIORITY_HIGH + for d in medium: + priorities[d] = max(priorities.get(d, 0), PRIORITY_MEDIUM) + for d in low: + priorities[d] = max(priorities.get(d, 0), PRIORITY_LOW) + + all_dates = sorted(priorities.keys()) + + if not all_dates: + logger.info("Scheduled booking scrape: no dates to scrape today") + return + + logger.info( + f"Scheduled booking scrape: queuing {len(all_dates)} dates " + f"(high={len(high)}, medium={len(medium)}, low={len(low)})" + ) + + # Populate queue + populate_queue(db, all_dates, priorities) + + # Process queue + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete(process_queue(db)) + if result.get('success'): + logger.info( + f"Scheduled booking scrape completed: " + f"{result.get('dates_completed', 0)} dates, " + f"{result.get('rates_scraped', 0)} rates" + ) + elif result.get('blocked'): + logger.warning( + f"Scheduled booking scrape blocked: {result.get('block_reason')}. " + f"Completed {result.get('dates_completed', 0)} dates. " + f"Remaining dates stay queued for retry." + ) + else: + logger.error(f"Scheduled booking scrape failed: {result.get('error')}") + finally: + loop.close() + + except Exception as e: + logger.error(f"Scheduled booking scrape error: {e}", exc_info=True) + finally: + db.close() + + +async def run_scheduled_booking_scrape_async(): + """Async wrapper for APScheduler.""" + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, run_scheduled_booking_scrape) diff --git a/backend/jobs/scrape_direct_rates.py b/backend/jobs/scrape_direct_rates.py new file mode 100644 index 0000000..f6b8b79 --- /dev/null +++ b/backend/jobs/scrape_direct_rates.py @@ -0,0 +1,43 @@ +""" +Daily job: scrape all enabled direct competitor hotels. +Hotels are scraped sequentially — the scraper enforces 10s delays per date +to avoid rate-limiting, so concurrent scraping is not beneficial. +""" +import logging +from sqlalchemy import text + +from database import SyncSessionLocal +from services.direct_scraper import run_scrape + +log = logging.getLogger(__name__) + + +def run_scrape_all_direct(): + db = SyncSessionLocal() + try: + rows = db.execute( + text("""SELECT id, name, profile_name, params + FROM direct_competitor_hotels + WHERE scrape_enabled = true + ORDER BY id""") + ).mappings().fetchall() + finally: + db.close() + + if not rows: + log.info("No direct competitor hotels enabled for scraping") + return + + log.info(f"Starting direct rate scrape for {len(rows)} hotels") + for hotel in rows: + try: + log.info(f"Scraping {hotel['name']} ({hotel['profile_name']})") + run_scrape( + hotel_id=hotel["id"], + profile_name=hotel["profile_name"], + params=hotel["params"], + ) + except Exception as e: + log.error(f"Direct scrape failed for hotel {hotel['id']} ({hotel['name']}): {e}") + + log.info("Direct rate scrape complete") diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 0000000..fa40673 --- /dev/null +++ b/backend/main.py @@ -0,0 +1,80 @@ +""" +Rate Monitor — FastAPI Backend +""" +import logging +import sys +from contextlib import asynccontextmanager + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[logging.StreamHandler(sys.stdout)] +) + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from database import DATABASE_URL +from api import bookability, competitors, analysis, direct +from scheduler import start_scheduler, shutdown_scheduler + + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + import os, psycopg2 + schema_path = os.path.join(os.path.dirname(__file__), 'schema.sql') + if os.path.exists(schema_path): + with open(schema_path) as f: + sql = f.read() + conn = psycopg2.connect(DATABASE_URL) + try: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute(sql) + logging.getLogger(__name__).info("Schema applied successfully") + finally: + conn.close() + except Exception as e: + logging.getLogger(__name__).warning(f"Schema init failed: {e}") + + try: + from services.booking_scraper import cleanup_stale_batches + from database import SyncSessionLocal + db = SyncSessionLocal() + try: + cleanup_stale_batches(db, max_age_minutes=10) + finally: + db.close() + except Exception as e: + logging.getLogger(__name__).warning(f"Stale batch cleanup failed: {e}") + + start_scheduler() + yield + shutdown_scheduler() + + +app = FastAPI( + title="Rate Monitor API", + description="Rate monitoring, competitor scraping, and bookability service", + version="1.0.0", + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(bookability.router, prefix="/bookability", tags=["Bookability"]) +app.include_router(competitors.router, prefix="/competitors", tags=["Competitors"]) +app.include_router(analysis.router, prefix="/analysis", tags=["Rate Analysis"]) +app.include_router(direct.router, prefix="/direct", tags=["Direct Rates"]) + + +@app.get("/health") +async def health_check(): + return {"status": "healthy", "service": "rate-monitor-api"} diff --git a/backend/migrate_direct_data.py b/backend/migrate_direct_data.py new file mode 100644 index 0000000..e0de24d --- /dev/null +++ b/backend/migrate_direct_data.py @@ -0,0 +1,167 @@ +""" +One-off migration: copy guestline-monitor SQLite data into PostgreSQL. +Run manually after deploy: + docker compose exec backend python migrate_direct_data.py +""" +import json +import os +import sqlite3 +import sys +from datetime import datetime, timezone + +import psycopg2 + +ARCHIVE_DIR = os.environ.get( + "GUESTLINE_ARCHIVE", + "/home/jtr/laptop-archive/guestline-monitor/data" +) +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://rates:rates_secret@localhost:5432/rates_db") + +CONFIG_DB = os.path.join(ARCHIVE_DIR, "config.db") + + +def dict_row(cursor, row): + return {col[0]: val for col, val in zip(cursor.description, row)} + + +def migrate(): + if not os.path.exists(CONFIG_DB): + print(f"ERROR: config.db not found at {CONFIG_DB}", file=sys.stderr) + print("Set GUESTLINE_ARCHIVE env var to the data directory path.") + sys.exit(1) + + pg = psycopg2.connect(DATABASE_URL) + pg.autocommit = False + + # ── Read hotel configs from SQLite ────────────────────────────────────── + src = sqlite3.connect(CONFIG_DB) + src.row_factory = dict_row + hotels = src.execute( + """SELECT id, name, profile, params, room_labels, rate_labels, + room_order, benchmark_room, benchmark_rate, + tier_base_room, tier_offsets, scrape_enabled, last_scraped_at + FROM hotels ORDER BY id""" + ).fetchall() + src.close() + + if not hotels: + print("No hotels found in config.db — nothing to migrate.") + return + + print(f"Migrating {len(hotels)} hotels...") + + # Map old SQLite hotel IDs to new PostgreSQL IDs + id_map: dict[int, int] = {} + + with pg.cursor() as cur: + for h in hotels: + params = h["params"] if isinstance(h["params"], str) else json.dumps(h["params"] or {}) + cur.execute( + """INSERT INTO direct_competitor_hotels + (name, profile_name, params, room_labels, rate_labels, room_order, + benchmark_room, benchmark_rate, tier_base_room, tier_offsets, + scrape_enabled, last_scraped_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id""", + ( + h["name"], + h["profile"], + params, + h["room_labels"] or "{}", + h["rate_labels"] or "{}", + h["room_order"] or "[]", + h["benchmark_room"], + h["benchmark_rate"], + h["tier_base_room"], + h["tier_offsets"] or "{}", + bool(h["scrape_enabled"]), + h["last_scraped_at"], + ) + ) + new_id = cur.fetchone()[0] + id_map[h["id"]] = new_id + print(f" Hotel {h['id']} → {new_id}: {h['name']}") + pg.commit() + + # ── Migrate snapshot data from per-hotel SQLite DBs ───────────────────── + total_rows = 0 + for old_id, new_id in id_map.items(): + db_path = os.path.join(ARCHIVE_DIR, f"hotel_{old_id}.db") + if not os.path.exists(db_path): + print(f" hotel_{old_id}.db not found — skipping snapshot data") + continue + + src = sqlite3.connect(db_path) + src.row_factory = dict_row + + # Migrate scrape_runs first + runs = src.execute( + "SELECT id, scraped_at, dates_found, rows_saved FROM scrape_runs ORDER BY id" + ).fetchall() + + run_id_map: dict[int, int] = {} + with pg.cursor() as cur: + for run in runs: + cur.execute( + """INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found, rows_saved) + VALUES (%s, %s, %s, %s) RETURNING id""", + (new_id, run["scraped_at"], run["dates_found"] or 0, run["rows_saved"] or 0) + ) + run_id_map[run["id"]] = cur.fetchone()[0] + pg.commit() + + # Migrate snapshots in batches + BATCH = 2000 + offset = 0 + hotel_rows = 0 + while True: + rows = src.execute( + """SELECT scrape_run_id, scraped_at, stay_date, room_id, rate_id, + availability, price_excl, price_incl, currency, min_stay_nights + FROM snapshots ORDER BY id LIMIT ? OFFSET ?""", + (BATCH, offset) + ).fetchall() + if not rows: + break + + with pg.cursor() as cur: + psycopg2.extras.execute_values( + cur, + """INSERT INTO direct_rates + (hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id, + availability, price_excl, price_incl, currency, min_stay_nights) + VALUES %s""", + [ + ( + new_id, + run_id_map.get(r["scrape_run_id"]), + r["scraped_at"], + r["stay_date"], + r["room_id"], + r["rate_id"], + r["availability"] or 0, + r["price_excl"], + r["price_incl"], + r["currency"] or "GBP", + r["min_stay_nights"], + ) + for r in rows + ] + ) + pg.commit() + + hotel_rows += len(rows) + offset += BATCH + print(f" {hotel_rows} rows migrated for hotel {old_id}...", end="\r") + + print(f" hotel_{old_id}: {hotel_rows} snapshot rows migrated") + total_rows += hotel_rows + src.close() + + pg.close() + print(f"\nMigration complete: {len(hotels)} hotels, {total_rows} snapshot rows.") + + +if __name__ == "__main__": + import psycopg2.extras + migrate() diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..6f48410 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.109.0 +uvicorn[standard]==0.27.0 +sqlalchemy==2.0.25 +asyncpg==0.29.0 +psycopg2-binary==2.9.9 +python-jose[cryptography]==3.3.0 +python-multipart==0.0.6 +httpx==0.26.0 +apscheduler==3.10.4 +pydantic==2.5.3 +pydantic-settings==2.1.0 +python-dotenv==1.0.0 +python-dateutil==2.8.2 +playwright>=1.40.0 diff --git a/backend/scheduler.py b/backend/scheduler.py new file mode 100644 index 0000000..d420f6d --- /dev/null +++ b/backend/scheduler.py @@ -0,0 +1,120 @@ +""" +APScheduler configuration for rate monitor jobs +""" +import logging +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.cron import CronTrigger +from sqlalchemy import text + +from database import SyncSessionLocal + +logger = logging.getLogger(__name__) + +scheduler = AsyncIOScheduler( + job_defaults={ + 'misfire_grace_time': 3600, + 'coalesce': True, + } +) + + +def get_config_value(key: str, default: str = None) -> str: + db = SyncSessionLocal() + try: + result = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = :key"), + {"key": key} + ) + row = result.fetchone() + if row and row.config_value: + return row.config_value + return default + except Exception as e: + logger.error(f"Error getting config {key}: {e}") + return default + finally: + db.close() + + +def is_sync_enabled(source: str) -> bool: + value = get_config_value(f"sync_{source}_enabled") + if value: + return value.lower() in ('true', '1', 'yes', 'enabled') + return False + + +def get_sync_time(source: str, default_hour: int = 5, default_minute: int = 0) -> tuple: + time_str = get_config_value(f"sync_{source}_time") + if time_str: + try: + parts = time_str.split(':') + return (int(parts[0]), int(parts[1])) + except (ValueError, IndexError): + pass + return (default_hour, default_minute) + + +async def run_scheduled_direct_scrape(): + from jobs.scrape_direct_rates import run_scrape_all_direct + import asyncio + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, run_scrape_all_direct) + + +async def run_scheduled_booking_scrape_async(): + from jobs.scrape_booking_rates import run_scheduled_booking_scrape + import asyncio + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, run_scheduled_booking_scrape) + + +async def run_scheduled_fetch_current_rates(): + if is_sync_enabled("newbook_current_rates"): + from jobs.fetch_current_rates import run_fetch_current_rates + import asyncio + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, run_fetch_current_rates) + else: + logger.debug("Newbook current rates sync skipped (disabled)") + + +def start_scheduler(): + # Booking.com scrape — daily at configurable time (default 05:30) + scrape_time = get_config_value('booking_scraper_daily_time', '05:30') + try: + h, m = scrape_time.split(':') + scrape_hour, scrape_minute = int(h), int(m) + except Exception: + scrape_hour, scrape_minute = 5, 30 + + scheduler.add_job( + run_scheduled_booking_scrape_async, + CronTrigger(hour=scrape_hour, minute=scrape_minute), + id='booking_scrape', + replace_existing=True, + ) + + # Newbook current rates fetch — daily at 05:20 + rates_hour, rates_minute = get_sync_time('newbook_current_rates', 5, 20) + scheduler.add_job( + run_scheduled_fetch_current_rates, + CronTrigger(hour=rates_hour, minute=rates_minute), + id='fetch_current_rates', + replace_existing=True, + ) + + # Direct booking engine scrape — daily at 06:00 + scheduler.add_job( + run_scheduled_direct_scrape, + CronTrigger(hour=6, minute=0), + id='scrape_direct_rates', + replace_existing=True, + ) + + scheduler.start() + logger.info(f"Scheduler started: booking scrape at {scrape_hour:02d}:{scrape_minute:02d}, rates fetch at {rates_hour:02d}:{rates_minute:02d}, direct scrape at 06:00") + + +def shutdown_scheduler(): + if scheduler.running: + scheduler.shutdown() diff --git a/backend/schema.sql b/backend/schema.sql new file mode 100644 index 0000000..71cec0e --- /dev/null +++ b/backend/schema.sql @@ -0,0 +1,267 @@ +-- Rate Monitor — Database Schema +-- All CREATE TABLE/INDEX are idempotent (IF NOT EXISTS) + +-- ============================================ +-- SYSTEM CONFIG +-- ============================================ + +CREATE TABLE IF NOT EXISTS system_config ( + config_key VARCHAR(100) PRIMARY KEY, + config_value TEXT, + description TEXT, + updated_at TIMESTAMPTZ DEFAULT NOW(), + updated_by VARCHAR(255) +); + +INSERT INTO system_config (config_key, config_value, description) VALUES +('booking_scraper_enabled', 'false', 'Enable automatic booking.com rate scraping (true/false)'), +('booking_scraper_paused', 'false', 'Scraper temporarily paused due to blocking (true/false)'), +('booking_scraper_pause_until', NULL, 'ISO datetime when pause expires'), +('booking_scraper_backend', 'playwright_local', 'Scraper backend: playwright_local | playwright_proxy | apify'), +('booking_scraper_daily_time', '05:30', 'Daily scrape time (HH:MM)'), +('booking_scraper_proxy_url', NULL, 'Proxy URL for playwright_proxy backend'), +('booking_scraper_proxy_username', NULL, 'Proxy username for playwright_proxy backend'), +('booking_scraper_proxy_password', NULL, 'Proxy password for playwright_proxy backend'), +('sync_newbook_current_rates_enabled', 'false', 'Enable automatic Newbook current rates sync (true/false)'), +('sync_newbook_current_rates_time', '05:20', 'Newbook current rates sync time (HH:MM)') +ON CONFLICT (config_key) DO NOTHING; + +-- ============================================ +-- BOOKING.COM SCRAPE LOG (batch tracking) +-- ============================================ + +CREATE TABLE IF NOT EXISTS booking_scrape_log ( + batch_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + scrape_type VARCHAR(50) NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + status VARCHAR(20) NOT NULL DEFAULT 'running', + hotels_found INTEGER DEFAULT 0, + rates_scraped INTEGER DEFAULT 0, + dates_queued INTEGER DEFAULT 0, + dates_completed INTEGER DEFAULT 0, + dates_failed INTEGER DEFAULT 0, + error_message TEXT, + blocked_at TIMESTAMPTZ, + resume_after TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_booking_scrape_log_started ON booking_scrape_log(started_at DESC); + +-- ============================================ +-- BOOKING.COM SCRAPE CONFIG +-- ============================================ + +CREATE TABLE IF NOT EXISTS booking_scrape_config ( + id SERIAL PRIMARY KEY, + location_name VARCHAR(255) NOT NULL, + location_search_url TEXT, + pages_to_scrape INTEGER DEFAULT 2, + adults INTEGER DEFAULT 2, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +-- ============================================ +-- BOOKING.COM SCRAPE QUEUE +-- ============================================ + +CREATE TABLE IF NOT EXISTS booking_scrape_queue ( + id SERIAL PRIMARY KEY, + rate_date DATE NOT NULL, + status VARCHAR(20) DEFAULT 'pending', + priority INTEGER DEFAULT 0, + attempts INTEGER DEFAULT 0, + max_attempts INTEGER DEFAULT 3, + last_attempt_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + error_message TEXT, + UNIQUE (rate_date, status) +); + +CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending + ON booking_scrape_queue(status, priority DESC, rate_date ASC) + WHERE status = 'pending'; + +-- ============================================ +-- BOOKING.COM HOTELS +-- ============================================ + +CREATE TABLE IF NOT EXISTS booking_com_hotels ( + id SERIAL PRIMARY KEY, + booking_com_id VARCHAR(50) UNIQUE, + name VARCHAR(255) NOT NULL, + booking_com_url TEXT, + star_rating DECIMAL(2,1), + review_score DECIMAL(3,1), + review_count INTEGER DEFAULT 0, + tier VARCHAR(20) DEFAULT 'market', -- own | competitor | market + display_order INTEGER DEFAULT 999, + notes TEXT, + is_active BOOLEAN DEFAULT TRUE, + first_seen_at TIMESTAMPTZ DEFAULT NOW(), + last_seen_at TIMESTAMPTZ DEFAULT NOW() +); + +-- ============================================ +-- BOOKING.COM RATES +-- ============================================ + +CREATE TABLE IF NOT EXISTS booking_com_rates ( + id SERIAL PRIMARY KEY, + hotel_id INTEGER NOT NULL REFERENCES booking_com_hotels(id) ON DELETE CASCADE, + rate_date DATE NOT NULL, + availability_status VARCHAR(30), -- available | sold_out | no_data + rate_gross DECIMAL(10,2), + currency VARCHAR(10) DEFAULT 'GBP', + room_type VARCHAR(100), + breakfast_included BOOLEAN DEFAULT FALSE, + free_cancellation BOOLEAN DEFAULT FALSE, + no_prepayment BOOLEAN DEFAULT FALSE, + rooms_left INTEGER, + scrape_batch_id UUID REFERENCES booking_scrape_log(batch_id), + scraped_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date); +CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date); +CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC); + +-- ============================================ +-- RATE PARITY ALERTS +-- ============================================ + +CREATE TABLE IF NOT EXISTS rate_parity_alerts ( + id SERIAL PRIMARY KEY, + rate_date DATE NOT NULL, + room_category VARCHAR(255), + newbook_rate DECIMAL(12,2), + booking_com_rate DECIMAL(12,2), + difference_pct DECIMAL(8,2), + alert_type VARCHAR(20), -- higher | lower + alert_status VARCHAR(20) DEFAULT 'active', -- active | acknowledged + created_at TIMESTAMPTZ DEFAULT NOW(), + acknowledged_at TIMESTAMPTZ, + acknowledged_by VARCHAR(255), + notes TEXT +); + +CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_date ON rate_parity_alerts(rate_date DESC); +CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_status ON rate_parity_alerts(alert_status); + +-- ============================================ +-- NEWBOOK CURRENT RATES (own hotel, for bookability display) +-- ============================================ + +CREATE TABLE IF NOT EXISTS newbook_current_rates ( + id SERIAL PRIMARY KEY, + category_id VARCHAR(50) NOT NULL, + rate_date DATE NOT NULL, + gross_rate DECIMAL(12,2), + net_rate DECIMAL(12,2), + tariffs_data JSONB DEFAULT '{}', + valid_from TIMESTAMP DEFAULT NOW(), + last_verified_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_current_rates_date ON newbook_current_rates(rate_date); +CREATE INDEX IF NOT EXISTS idx_current_rates_category ON newbook_current_rates(category_id); +CREATE INDEX IF NOT EXISTS idx_current_rates_tariffs ON newbook_current_rates USING gin(tariffs_data); +CREATE INDEX IF NOT EXISTS idx_current_rates_latest ON newbook_current_rates(category_id, rate_date, valid_from DESC); + +-- ============================================ +-- NEWBOOK ROOM CATEGORIES (for bookability display) +-- ============================================ + +CREATE TABLE IF NOT EXISTS newbook_room_categories ( + id SERIAL PRIMARY KEY, + site_id VARCHAR(50) NOT NULL UNIQUE, + site_name VARCHAR(255) NOT NULL, + site_type VARCHAR(100), + room_count INTEGER DEFAULT 0, + is_included BOOLEAN DEFAULT TRUE, + display_order INTEGER DEFAULT 0, + fetched_at TIMESTAMP DEFAULT NOW() +); + +-- ============================================ +-- NEWBOOK OCCUPANCY REPORT DATA +-- ============================================ + +CREATE TABLE IF NOT EXISTS newbook_occupancy_report_data ( + id SERIAL PRIMARY KEY, + date DATE NOT NULL, + category_id VARCHAR(50) NOT NULL, + category_name VARCHAR(255), + available INTEGER DEFAULT 0, + occupied INTEGER DEFAULT 0, + maintenance INTEGER DEFAULT 0, + allotted INTEGER DEFAULT 0, + revenue_gross DECIMAL(12,2) DEFAULT 0, + revenue_net DECIMAL(12,2) DEFAULT 0, + occupancy_pct DECIMAL(5,2), + fetched_at TIMESTAMP DEFAULT NOW(), + UNIQUE(date, category_id) +); + +CREATE INDEX IF NOT EXISTS idx_occupancy_report_date ON newbook_occupancy_report_data(date); + +-- ============================================ +-- DIRECT COMPETITOR HOTEL CONFIGS +-- ============================================ + +CREATE TABLE IF NOT EXISTS direct_competitor_hotels ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + profile_name VARCHAR(50) NOT NULL, + params JSONB NOT NULL DEFAULT '{}', + room_labels JSONB DEFAULT '{}', + rate_labels JSONB DEFAULT '{}', + room_order JSONB DEFAULT '[]', + benchmark_room VARCHAR(100), + benchmark_rate VARCHAR(100), + tier_base_room VARCHAR(100), + tier_offsets JSONB DEFAULT '{}', + scrape_enabled BOOLEAN DEFAULT TRUE, + last_scraped_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +-- ============================================ +-- DIRECT SCRAPE RUNS +-- ============================================ + +CREATE TABLE IF NOT EXISTS direct_scrape_runs ( + id SERIAL PRIMARY KEY, + hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE, + scraped_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + dates_found INTEGER DEFAULT 0, + rows_saved INTEGER DEFAULT 0 +); + +-- ============================================ +-- DIRECT RATES (per-room snapshots from booking engines) +-- ============================================ + +CREATE TABLE IF NOT EXISTS direct_rates ( + id SERIAL PRIMARY KEY, + hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE, + scrape_run_id INTEGER REFERENCES direct_scrape_runs(id), + scraped_at TIMESTAMPTZ NOT NULL, + stay_date DATE NOT NULL, + room_id VARCHAR(100) NOT NULL, + rate_id VARCHAR(100) NOT NULL, + availability INTEGER DEFAULT 0, + price_excl DECIMAL(10,2), + price_incl DECIMAL(10,2), + currency VARCHAR(10) DEFAULT 'GBP', + min_stay_nights INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_direct_rates_hotel_date ON direct_rates(hotel_id, stay_date); +CREATE INDEX IF NOT EXISTS idx_direct_rates_scraped ON direct_rates(scraped_at DESC); + +-- Link booking_com_hotels to direct_competitor_hotels (optional, for Market View direct column) +ALTER TABLE booking_com_hotels ADD COLUMN IF NOT EXISTS direct_hotel_id INTEGER + REFERENCES direct_competitor_hotels(id) ON DELETE SET NULL; diff --git a/backend/services/__init__.py b/backend/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py new file mode 100644 index 0000000..a3bd9ce --- /dev/null +++ b/backend/services/booking_scraper.py @@ -0,0 +1,829 @@ +""" +Booking.com Rate Scraper Service + +Main service for scraping competitor rates from booking.com. +Uses pluggable backends (Playwright local, proxy, Apify) via factory pattern. + +Features: +- Location-based search (1 query = 40+ hotels) +- Hotel discovery and tier management +- Rate extraction with availability status +- Anti-scrape detection and pause/resume +""" + +import logging +import uuid +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import List, Optional, Dict, Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus + +logger = logging.getLogger(__name__) + + +def get_scraper_backend(db: Session) -> ScraperBackend: + """ + Factory to get configured scraper backend. + + Reads backend type from system_config and returns appropriate instance. + """ + # Get backend configuration + result = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'") + ).fetchone() + + backend_type = result.config_value if result and result.config_value else 'playwright_local' + + if backend_type == 'playwright_local': + return PlaywrightLocalBackend() + + elif backend_type == 'playwright_proxy': + # Get proxy config + proxy_result = db.execute( + text(""" + SELECT config_key, config_value FROM system_config + WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password') + """) + ) + proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()} + return PlaywrightLocalBackend(proxy_config=proxy_config) + + elif backend_type == 'apify': + # Future: Apify backend + raise NotImplementedError("Apify backend not yet implemented") + + else: + logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local") + return PlaywrightLocalBackend() + + +def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]: + """Get the active scrape location configuration.""" + result = db.execute( + text(""" + SELECT id, location_name, location_search_url, pages_to_scrape, adults + FROM booking_scrape_config + WHERE is_active = TRUE + ORDER BY id + LIMIT 1 + """) + ).fetchone() + + if not result: + return None + + return { + 'id': result.id, + 'location_name': result.location_name, + 'location_search_url': result.location_search_url, + 'pages_to_scrape': result.pages_to_scrape or 2, + 'adults': result.adults or 2, + } + + +async def is_scraper_paused(db: Session) -> bool: + """Check if scraper is currently paused due to blocking.""" + result = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'") + ).fetchone() + + if not result or result.config_value != 'true': + return False + + # Check if pause period has expired + pause_until_result = db.execute( + text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'") + ).fetchone() + + if pause_until_result and pause_until_result.config_value: + try: + pause_until = datetime.fromisoformat(pause_until_result.config_value) + if datetime.now() >= pause_until: + # Pause expired, reset + db.execute( + text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'") + ) + db.commit() + return False + except ValueError: + pass + + return True + + +async def set_scraper_paused(db: Session, paused: bool, hours: int = 2): + """Set scraper pause status.""" + db.execute( + text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"), + {'val': 'true' if paused else 'false'} + ) + if paused: + pause_until = (datetime.now() + timedelta(hours=hours)).isoformat() + db.execute( + text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"), + {'val': pause_until} + ) + db.commit() + + +def save_hotel(db: Session, hotel: HotelData) -> int: + """ + Save or update a hotel in the database. + + Returns the hotel's database ID. + """ + # Check if hotel exists + existing = db.execute( + text("SELECT id FROM booking_com_hotels WHERE booking_com_id = :bid"), + {'bid': hotel.booking_com_id} + ).fetchone() + + if existing: + # Update last_seen_at and any changed fields + db.execute( + text(""" + UPDATE booking_com_hotels SET + name = COALESCE(:name, name), + booking_com_url = COALESCE(:url, booking_com_url), + star_rating = COALESCE(:stars, star_rating), + review_score = COALESCE(:score, review_score), + review_count = COALESCE(:count, review_count), + last_seen_at = NOW() + WHERE booking_com_id = :bid + """), + { + 'bid': hotel.booking_com_id, + 'name': hotel.name, + 'url': hotel.booking_com_url, + 'stars': float(hotel.star_rating) if hotel.star_rating else None, + 'score': float(hotel.review_score) if hotel.review_score else None, + 'count': hotel.review_count, + } + ) + return existing.id + else: + # Insert new hotel (default tier is 'market') + result = db.execute( + text(""" + INSERT INTO booking_com_hotels + (booking_com_id, name, booking_com_url, star_rating, review_score, review_count, tier) + VALUES (:bid, :name, :url, :stars, :score, :count, 'market') + RETURNING id + """), + { + 'bid': hotel.booking_com_id, + 'name': hotel.name, + 'url': hotel.booking_com_url, + 'stars': float(hotel.star_rating) if hotel.star_rating else None, + 'score': float(hotel.review_score) if hotel.review_score else None, + 'count': hotel.review_count, + } + ) + return result.fetchone().id + + +def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID): + """Save a rate to the database.""" + db.execute( + text(""" + INSERT INTO booking_com_rates + (hotel_id, rate_date, availability_status, rate_gross, currency, room_type, + breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id) + VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type, + :breakfast, :cancel, :prepay, :rooms_left, :batch_id) + """), + { + 'hotel_id': hotel_id, + 'rate_date': rate.rate_date, + 'status': rate.availability_status.value, + 'rate': float(rate.rate_gross) if rate.rate_gross else None, + 'currency': rate.currency, + 'room_type': rate.room_type, + 'breakfast': rate.breakfast_included, + 'cancel': rate.free_cancellation, + 'prepay': rate.no_prepayment, + 'rooms_left': rate.rooms_left, + 'batch_id': str(batch_id), + } + ) + + +def create_scrape_batch(db: Session, scrape_type: str) -> uuid.UUID: + """Create a new scrape batch log entry.""" + batch_id = uuid.uuid4() + db.execute( + text(""" + INSERT INTO booking_scrape_log + (batch_id, scrape_type, started_at, status) + VALUES (:batch_id, :scrape_type, NOW(), 'running') + """), + {'batch_id': str(batch_id), 'scrape_type': scrape_type} + ) + db.commit() + return batch_id + + +def update_scrape_batch( + db: Session, + batch_id: uuid.UUID, + status: str, + hotels_found: int = 0, + rates_scraped: int = 0, + error_message: str = None, + blocked: bool = False +): + """Update scrape batch log with results.""" + db.execute( + text(""" + UPDATE booking_scrape_log SET + completed_at = CASE WHEN :status IN ('completed', 'failed', 'blocked') THEN NOW() ELSE NULL END, + status = :status, + hotels_found = :hotels, + rates_scraped = :rates, + error_message = :error, + blocked_at = CASE WHEN :blocked THEN NOW() ELSE NULL END, + resume_after = CASE WHEN :blocked THEN NOW() + INTERVAL '2 hours' ELSE NULL END + WHERE batch_id = :batch_id + """), + { + 'batch_id': str(batch_id), + 'status': status, + 'hotels': hotels_found, + 'rates': rates_scraped, + 'error': error_message, + 'blocked': blocked, + } + ) + db.commit() + + +def cleanup_stale_batches(db: Session, max_age_minutes: int = 60): + """ + Mark any 'running' scrape batches as 'failed' if they've been running + longer than max_age_minutes. This handles orphaned batches from + container restarts or crashes. + """ + result = db.execute( + text(""" + UPDATE booking_scrape_log SET + status = 'failed', + completed_at = NOW(), + error_message = 'Interrupted (container restart or timeout)' + WHERE status = 'running' + AND started_at < NOW() - INTERVAL ':mins minutes' + RETURNING batch_id + """.replace(':mins', str(int(max_age_minutes)))) + ) + cleaned = result.fetchall() + db.commit() + if cleaned: + logger.info(f"Cleaned up {len(cleaned)} stale running scrape batch(es)") + return len(cleaned) + + +async def scrape_date( + db: Session, + rate_date: date, + backend: ScraperBackend, + config: Dict[str, Any], + batch_id: uuid.UUID +) -> Dict[str, Any]: + """ + Scrape rates for a single date. + + Args: + db: Database session + rate_date: Date to scrape rates for + backend: Scraper backend instance + config: Scrape configuration + batch_id: Current batch ID + + Returns: + Dict with 'success', 'blocked', 'hotels_count', 'rates_count' + """ + check_in = rate_date + check_out = rate_date + timedelta(days=1) # Single night + + result = await backend.scrape_location_search( + location=config['location_name'], + check_in=check_in, + check_out=check_out, + adults=config['adults'], + pages=config['pages_to_scrape'] + ) + + if result.blocked: + return { + 'success': False, + 'blocked': True, + 'block_reason': result.block_reason, + 'hotels_count': 0, + 'rates_count': 0, + } + + if not result.success: + return { + 'success': False, + 'blocked': False, + 'error': result.error_message, + 'hotels_count': 0, + 'rates_count': 0, + } + + # Save hotels and rates + hotels_saved = 0 + rates_saved = 0 + + for hotel, rate in zip(result.hotels, result.rates): + if not hotel.booking_com_id: + continue + + try: + hotel_id = save_hotel(db, hotel) + save_rate(db, rate, hotel_id, batch_id) + hotels_saved += 1 + rates_saved += 1 + except Exception as e: + logger.warning(f"Error saving hotel/rate: {e}") + continue + + db.commit() + + return { + 'success': True, + 'blocked': False, + 'hotels_count': hotels_saved, + 'rates_count': rates_saved, + } + + +async def run_manual_scrape( + db: Session, + from_date: date, + to_date: date = None +) -> Dict[str, Any]: + """ + Run a manual scrape for testing/on-demand use. + + Args: + db: Database session + from_date: Start date + to_date: End date (defaults to from_date for single day) + + Returns: + Dict with scrape results summary + """ + if to_date is None: + to_date = from_date + + # Check if paused + if await is_scraper_paused(db): + return { + 'success': False, + 'error': 'Scraper is currently paused due to blocking. Try again later.', + } + + # Get config + config = get_scrape_config(db) + if not config: + return { + 'success': False, + 'error': 'No scrape location configured. Add a location in settings.', + } + + # Create batch + batch_id = create_scrape_batch(db, 'manual') + + # Get backend + backend = get_scraper_backend(db) + + total_hotels = 0 + total_rates = 0 + dates_completed = 0 + dates_failed = 0 + + try: + current_date = from_date + while current_date <= to_date: + logger.info(f"Scraping date: {current_date}") + + result = await scrape_date(db, current_date, backend, config, batch_id) + + if result['blocked']: + # Blocking detected - pause and exit + await set_scraper_paused(db, True, hours=2) + update_scrape_batch( + db, batch_id, + status='blocked', + hotels_found=total_hotels, + rates_scraped=total_rates, + error_message=f"Blocked: {result.get('block_reason', 'unknown')}", + blocked=True + ) + return { + 'success': False, + 'blocked': True, + 'block_reason': result.get('block_reason'), + 'dates_completed': dates_completed, + 'dates_failed': dates_failed, + 'hotels_found': total_hotels, + 'rates_scraped': total_rates, + } + + if result['success']: + total_hotels += result['hotels_count'] + total_rates += result['rates_count'] + dates_completed += 1 + else: + dates_failed += 1 + logger.warning(f"Failed to scrape {current_date}: {result.get('error')}") + + current_date += timedelta(days=1) + + # Update batch as completed + update_scrape_batch( + db, batch_id, + status='completed', + hotels_found=total_hotels, + rates_scraped=total_rates + ) + + return { + 'success': True, + 'blocked': False, + 'dates_completed': dates_completed, + 'dates_failed': dates_failed, + 'hotels_found': total_hotels, + 'rates_scraped': total_rates, + } + + except Exception as e: + logger.error(f"Scrape error: {e}") + update_scrape_batch( + db, batch_id, + status='failed', + hotels_found=total_hotels, + rates_scraped=total_rates, + error_message=str(e) + ) + return { + 'success': False, + 'error': str(e), + 'dates_completed': dates_completed, + 'dates_failed': dates_failed, + 'hotels_found': total_hotels, + 'rates_scraped': total_rates, + } + finally: + await backend.close() + + +# ============================================ +# QUEUE MANAGEMENT +# ============================================ + +def populate_queue(db: Session, dates: List[date], priorities: Dict[date, int] = None): + """ + Add dates to the scrape queue, skipping any already pending/processing. + + Args: + db: Database session + dates: Dates to add to the queue + priorities: Optional priority map (higher = scraped first). Default: 0 + """ + if not dates: + return 0 + + added = 0 + for rate_date in dates: + priority = (priorities or {}).get(rate_date, 0) + try: + db.execute( + text(""" + INSERT INTO booking_scrape_queue (rate_date, status, priority) + VALUES (:rate_date, 'pending', :priority) + ON CONFLICT (rate_date, status) DO UPDATE SET + priority = GREATEST(booking_scrape_queue.priority, :priority) + """), + {'rate_date': rate_date, 'priority': priority} + ) + added += 1 + except Exception: + # Ignore duplicates or constraint issues + pass + + db.commit() + logger.info(f"Queue: added/updated {added} dates") + return added + + +def get_pending_queue_items(db: Session, limit: int = 50) -> List[Dict[str, Any]]: + """Get pending queue items ordered by priority (highest first), then date.""" + result = db.execute( + text(""" + SELECT id, rate_date, priority, attempts, max_attempts + FROM booking_scrape_queue + WHERE status = 'pending' AND attempts < max_attempts + ORDER BY priority DESC, rate_date ASC + LIMIT :limit + """), + {'limit': limit} + ) + return [dict(row._mapping) for row in result.fetchall()] + + +def mark_queue_item(db: Session, queue_id: int, status: str, error_message: str = None): + """Update a queue item's status.""" + if status == 'completed': + db.execute( + text(""" + UPDATE booking_scrape_queue SET + status = 'completed', + completed_at = NOW(), + last_attempt_at = NOW(), + attempts = attempts + 1 + WHERE id = :id + """), + {'id': queue_id} + ) + elif status == 'failed': + db.execute( + text(""" + UPDATE booking_scrape_queue SET + status = CASE + WHEN attempts + 1 >= max_attempts THEN 'failed' + ELSE 'pending' + END, + last_attempt_at = NOW(), + attempts = attempts + 1, + error_message = :error + WHERE id = :id + """), + {'id': queue_id, 'error': error_message} + ) + db.commit() + + +def clear_old_queue_items(db: Session, days: int = 7): + """Remove completed/failed queue items older than N days.""" + db.execute( + text(""" + DELETE FROM booking_scrape_queue + WHERE status IN ('completed', 'failed') + AND created_at < NOW() - INTERVAL ':days days' + """.replace(':days', str(int(days)))) + ) + db.commit() + + +async def process_queue(db: Session) -> Dict[str, Any]: + """ + Process pending items from the scrape queue. + + Picks up pending items in priority order, scrapes each date, + and handles blocking/retries. + + Returns: + Dict with processing results + """ + # Check if paused + if await is_scraper_paused(db): + return { + 'success': False, + 'error': 'Scraper is currently paused due to blocking.', + } + + # Get config + config = get_scrape_config(db) + if not config: + return { + 'success': False, + 'error': 'No scrape location configured.', + } + + # Get pending items + items = get_pending_queue_items(db, limit=200) + if not items: + return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'} + + # Create batch + batch_id = create_scrape_batch(db, 'scheduled') + + # Update batch with queue count + db.execute( + text("UPDATE booking_scrape_log SET dates_queued = :count WHERE batch_id = :bid"), + {'count': len(items), 'bid': str(batch_id)} + ) + db.commit() + + # Get backend + backend = get_scraper_backend(db) + + total_hotels = 0 + total_rates = 0 + dates_completed = 0 + dates_failed = 0 + + try: + for item in items: + rate_date = item['rate_date'] + queue_id = item['id'] + + logger.info(f"Queue processing: {rate_date} (priority={item['priority']}, attempt={item['attempts']+1})") + + result = await scrape_date(db, rate_date, backend, config, batch_id) + + if result['blocked']: + # Mark this item as failed, pause, and stop + mark_queue_item(db, queue_id, 'failed', f"Blocked: {result.get('block_reason')}") + await set_scraper_paused(db, True, hours=2) + update_scrape_batch( + db, batch_id, + status='blocked', + hotels_found=total_hotels, + rates_scraped=total_rates, + error_message=f"Blocked: {result.get('block_reason', 'unknown')}", + blocked=True + ) + # Update dates counters + db.execute( + text(""" + UPDATE booking_scrape_log SET + dates_completed = :completed, + dates_failed = :failed + WHERE batch_id = :bid + """), + {'completed': dates_completed, 'failed': dates_failed + 1, 'bid': str(batch_id)} + ) + db.commit() + return { + 'success': False, + 'blocked': True, + 'block_reason': result.get('block_reason'), + 'dates_completed': dates_completed, + 'dates_failed': dates_failed + 1, + 'hotels_found': total_hotels, + 'rates_scraped': total_rates, + } + + if result['success']: + mark_queue_item(db, queue_id, 'completed') + total_hotels += result['hotels_count'] + total_rates += result['rates_count'] + dates_completed += 1 + else: + mark_queue_item(db, queue_id, 'failed', result.get('error')) + dates_failed += 1 + logger.warning(f"Queue: failed to scrape {rate_date}: {result.get('error')}") + + # Update batch as completed + update_scrape_batch( + db, batch_id, + status='completed', + hotels_found=total_hotels, + rates_scraped=total_rates + ) + db.execute( + text(""" + UPDATE booking_scrape_log SET + dates_completed = :completed, + dates_failed = :failed + WHERE batch_id = :bid + """), + {'completed': dates_completed, 'failed': dates_failed, 'bid': str(batch_id)} + ) + db.commit() + + return { + 'success': True, + 'blocked': False, + 'dates_completed': dates_completed, + 'dates_failed': dates_failed, + 'hotels_found': total_hotels, + 'rates_scraped': total_rates, + } + + except Exception as e: + logger.error(f"Queue processing error: {e}") + update_scrape_batch( + db, batch_id, + status='failed', + hotels_found=total_hotels, + rates_scraped=total_rates, + error_message=str(e) + ) + return { + 'success': False, + 'error': str(e), + 'dates_completed': dates_completed, + 'dates_failed': dates_failed, + } + finally: + await backend.close() + + +def get_competitor_matrix( + db: Session, + from_date: date, + to_date: date, + include_market: bool = False +) -> List[Dict[str, Any]]: + """ + Get rate comparison matrix for competitors. + + Args: + db: Database session + from_date: Start date + to_date: End date + include_market: Include market tier hotels + + Returns: + List of rate records for matrix display + """ + tier_filter = "h.tier IN ('own', 'competitor')" + if include_market: + tier_filter = "h.tier IN ('own', 'competitor', 'market')" + + result = db.execute( + text(f""" + SELECT + r.rate_date, + h.id AS hotel_id, + h.name AS hotel_name, + h.tier, + h.display_order, + h.star_rating, + h.review_score, + r.availability_status, + r.rate_gross, + r.room_type, + r.breakfast_included, + r.free_cancellation, + r.no_prepayment, + r.rooms_left, + r.scraped_at + FROM booking_latest_rates r + JOIN booking_com_hotels h ON r.hotel_id = h.id + WHERE {tier_filter} + AND h.is_active = TRUE + AND r.rate_date BETWEEN :from_date AND :to_date + ORDER BY r.rate_date, h.display_order, h.name + """), + {'from_date': from_date, 'to_date': to_date} + ) + + return [dict(row._mapping) for row in result.fetchall()] + + +def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]: + """ + Get list of discovered hotels. + + Args: + db: Database session + tier: Filter by tier ('own', 'competitor', 'market') or None for all + + Returns: + List of hotel records + """ + where_clause = "WHERE is_active = TRUE" + if tier: + where_clause += f" AND tier = '{tier}'" + + result = db.execute( + text(f""" + SELECT + id, booking_com_id, name, booking_com_url, + star_rating, review_score, review_count, + tier, display_order, notes, + first_seen_at, last_seen_at + FROM booking_com_hotels + {where_clause} + ORDER BY display_order, name + """) + ) + + return [dict(row._mapping) for row in result.fetchall()] + + +def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None): + """Update a hotel's tier and display order.""" + if tier not in ('own', 'competitor', 'market'): + raise ValueError(f"Invalid tier: {tier}") + + params = {'hotel_id': hotel_id, 'tier': tier} + set_clause = "tier = :tier" + + if display_order is not None: + set_clause += ", display_order = :order" + params['order'] = display_order + + db.execute( + text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"), + params + ) + db.commit() diff --git a/backend/services/central_settings.py b/backend/services/central_settings.py new file mode 100644 index 0000000..6814b09 --- /dev/null +++ b/backend/services/central_settings.py @@ -0,0 +1,100 @@ +""" +Client for the stack's central Settings service. + +NewBook credentials are managed once in the Settings app (LXC 116) and +fetched live by every app — the same pattern as cashup / room-planner / +maintenance (see their lib/newbook.js). Falls back to None if the service +is unreachable so callers can fall back to app-local config. +""" +import logging +import os +import time +from typing import Optional + +import httpx + +logger = logging.getLogger(__name__) + +SETTINGS_URL = os.getenv("SETTINGS_URL", "") +SETTINGS_SECRET = os.getenv("SETTINGS_SECRET", "") + +_CACHE_TTL = 60 # seconds — credentials change rarely; avoid hammering the service +_cache: dict = {} + + +async def get_integration(name: str) -> Optional[dict]: + """ + Fetch integration config (e.g. 'newbook') from the central Settings + service. Returns the config dict, or None if unavailable/unconfigured. + """ + if not SETTINGS_URL or not SETTINGS_SECRET: + return None + + cached = _cache.get(name) + if cached and (time.monotonic() - cached[0]) < _CACHE_TTL: + return cached[1] + + url = f"{SETTINGS_URL}/settings/api/internal/integration/{name}" + try: + async with httpx.AsyncClient(timeout=5.0) as client: + resp = await client.get( + url, headers={"Authorization": f"Bearer {SETTINGS_SECRET}"} + ) + resp.raise_for_status() + data = resp.json() + _cache[name] = (time.monotonic(), data) + return data + except Exception as e: + logger.warning(f"Central settings fetch failed for '{name}': {e}") + return None + + +def _extract_newbook(s: Optional[dict]) -> Optional[dict]: + if not s: + return None + creds = { + "api_key": s.get("api_key") or "", + "username": s.get("username") or "", + "password": s.get("password") or "", + "region": s.get("region") or "eu", + } + # Only usable if the essential fields are present + if not (creds["api_key"] and creds["username"] and creds["password"]): + return None + return creds + + +async def get_newbook_credentials() -> Optional[dict]: + """ + Returns {'api_key', 'username', 'password', 'region'} from central + settings, or None if not available (caller should fall back). + """ + return _extract_newbook(await get_integration("newbook")) + + +def get_integration_sync(name: str) -> Optional[dict]: + """Blocking variant of get_integration for sync job contexts.""" + if not SETTINGS_URL or not SETTINGS_SECRET: + return None + + cached = _cache.get(name) + if cached and (time.monotonic() - cached[0]) < _CACHE_TTL: + return cached[1] + + url = f"{SETTINGS_URL}/settings/api/internal/integration/{name}" + try: + resp = httpx.get( + url, headers={"Authorization": f"Bearer {SETTINGS_SECRET}"}, timeout=5.0 + ) + resp.raise_for_status() + data = resp.json() + _cache[name] = (time.monotonic(), data) + return data + except Exception as e: + logger.warning(f"Central settings fetch failed for '{name}': {e}") + return None + + +def get_newbook_credentials_sync() -> Optional[dict]: + """Blocking variant of get_newbook_credentials for sync job contexts.""" + return _extract_newbook(get_integration_sync("newbook")) diff --git a/backend/services/direct_profiles/__init__.py b/backend/services/direct_profiles/__init__.py new file mode 100644 index 0000000..bb2f651 --- /dev/null +++ b/backend/services/direct_profiles/__init__.py @@ -0,0 +1,28 @@ +from .guestline import GuestlineProfile +from .newbook_scrape import NewbookScrapeProfile +from .travelclick import TravelClickProfile +from .directbook import DirectBookProfile +from .base import BaseProfile + +PROFILES = { + "guestline": GuestlineProfile, + "newbook_scrape": NewbookScrapeProfile, + "travelclick": TravelClickProfile, + "directbook": DirectBookProfile, +} + + +def get_profile(name: str) -> BaseProfile: + cls = PROFILES.get(name) + if not cls: + raise ValueError(f"Unknown engine profile: {name}") + return cls() + + +def detect_profile(url: str) -> dict | None: + """Given a booking URL, return suggested profile name and extracted params.""" + for name, cls in PROFILES.items(): + result = cls.detect(url) + if result is not None: + return {"profile": name, **result} + return None diff --git a/backend/services/direct_profiles/base.py b/backend/services/direct_profiles/base.py new file mode 100644 index 0000000..cd0209c --- /dev/null +++ b/backend/services/direct_profiles/base.py @@ -0,0 +1,25 @@ +from abc import ABC, abstractmethod + + +class BaseProfile(ABC): + name: str = "" + label: str = "" + # Fields required to configure this engine, shown in the add-hotel form + # Each entry: {"key": "hotel_id", "label": "Hotel ID", "help": "e.g. THREEWAYS"} + required_params: list[dict] = [] + + @classmethod + @abstractmethod + def detect(cls, url: str) -> dict | None: + """Return extracted params dict if URL matches this engine, else None.""" + + @abstractmethod + async def fetch_arrival_dates(self, client, params: dict) -> list[str]: + """Return list of bookable arrival date strings YYYY-MM-DD.""" + + @abstractmethod + async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]: + """Return list of room/rate dicts for the stay. + Each dict must have: roomId, rateId, availability, prices[{amountBeforeTax, amountAfterTax}], currencyCode + prices list has one entry per night when nights > 1. + """ diff --git a/backend/services/direct_profiles/directbook.py b/backend/services/direct_profiles/directbook.py new file mode 100644 index 0000000..c9d595a --- /dev/null +++ b/backend/services/direct_profiles/directbook.py @@ -0,0 +1,175 @@ +import json +import re +from datetime import date, timedelta +from urllib.parse import quote +from .base import BaseProfile + +API_BASE = "https://direct-book.com" + +SETTINGS_HASH = "52d786f5c45232c8c16022bc3af6dab2e1994f4919b953afafe188095125e9b6" +QUOTESETS_HASH = "1012a6203854357e44786380240eefad6b2ad863aee6ba79748c81a851f29217" +ROOMTYPES_HASH = "8021345a2e1f993717b1960097489b456a6b2dc136990b6f919adba1b1fe2c1f" + +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Accept": "application/json", + # Required to bypass Apollo CSRF protection on the /api/graphql endpoint + "Content-Type": "application/json", +} + + +def _graphql_url(operation: str, variables: dict, sha256: str) -> str: + return ( + f"{API_BASE}/api/graphql" + f"?operationName={operation}" + f"&variables={_json_compact(variables)}" + f"&extensions={_json_compact({'persistedQuery': {'version': 1, 'sha256Hash': sha256}})}" + ) + + +def _json_compact(obj) -> str: + return quote(json.dumps(obj, separators=(',', ':')), safe='') + + +async def _get_property_id(client, channel_code: str) -> str: + """Fetch numeric propertyId from settings query.""" + url = _graphql_url("settings", {"channelCode": channel_code}, SETTINGS_HASH) + r = await client.get(url, headers=HEADERS, timeout=20) + r.raise_for_status() + return str(r.json()["data"]["settings"]["uuid"]) + + +class DirectBookProfile(BaseProfile): + name = "directbook" + label = "SiteMinder Direct Book" + required_params = [ + {"key": "channel_code", "label": "Channel Code", + "help": "The property slug in the booking URL, e.g. 'grapevinestowdirect'"}, + ] + + @classmethod + def detect(cls, url: str) -> dict | None: + m = re.search(r'direct-book\.com/properties/([^/?#\s]+)', url) + if m: + return {"channel_code": m.group(1)} + return None + + async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]: + channel_code = params["channel_code"] + url = _graphql_url( + "roomTypes", + {"channelCode": channel_code, "checkInDate": date.today().isoformat(), + "checkOutDate": (date.today() + timedelta(days=1)).isoformat(), "locale": "en"}, + ROOMTYPES_HASH, + ) + r = await client.get(url, headers=HEADERS, timeout=30) + r.raise_for_status() + room_types = r.json()["data"]["roomTypes"] + + room_labels = {rt["uuid"]: rt["name"] for rt in room_types} + rate_labels = {} + for rt in room_types: + for rate in rt.get("rates", []): + rate_labels[rate["uuid"]] = rate["name"] + return room_labels, rate_labels + + async def fetch_arrival_dates(self, client, params: dict) -> list[str]: + channel_code = params["channel_code"] + today = date.today() + + # Fetch 12 months of availability in monthly chunks (API seems to accept wide ranges too) + end = today.replace(day=1) + timedelta(days=365) + # Build monthly windows to stay within API limits + available_dates: list[str] = [] + current = today.replace(day=1) + while current <= end: + # Last day of the month + if current.month == 12: + month_end = current.replace(year=current.year + 1, month=1, day=1) - timedelta(days=1) + else: + month_end = current.replace(month=current.month + 1, day=1) - timedelta(days=1) + + check_from = max(today, current).isoformat() + check_to = month_end.isoformat() + + url = ( + f"{API_BASE}/api/properties/{channel_code}/availability" + f"?checkInsFrom={check_from}&checkInsTo={check_to}" + ) + try: + r = await client.get(url, headers=HEADERS, timeout=20) + r.raise_for_status() + for entry in r.json().get("result", []): + if entry.get("canCheckIn"): + d = entry["date"][:10] + if d >= today.isoformat(): + available_dates.append(d) + except Exception: + pass + + # Advance to next month + if current.month == 12: + current = current.replace(year=current.year + 1, month=1) + else: + current = current.replace(month=current.month + 1) + + return sorted(set(available_dates)) + + async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]: + channel_code = params["channel_code"] + + # Get propertyId (numeric) — cache it on the params dict to avoid repeat fetches + if "property_id" not in params: + params["property_id"] = await _get_property_id(client, channel_code) + property_id = int(params["property_id"]) + + departure = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat() + + url = _graphql_url( + "quoteSets", + { + "propertyId": property_id, + "promocode": "", + "checkInDate": arrival, + "checkOutDate": departure, + "adults": 2, + "children": 0, + "infants": 0, + "currencyFrom": "GBP", + "currencyTo": "GBP", + }, + QUOTESETS_HASH, + ) + r = await client.get(url, headers=HEADERS, timeout=20) + if r.status_code == 404: + return [] + r.raise_for_status() + + rooms = [] + for qs in r.json()["data"].get("quoteSets", []): + room_id = str(qs["roomTypeId"]) + for quote in qs.get("quotes", []): + rate_id = str(quote["roomRateId"]) + price = quote["price"]["amount"] + available = quote.get("available", 0) + + # Build per-night prices from breakdown if multi-night + breakdown = quote.get("breakdown", []) + if breakdown: + prices = [ + {"amountBeforeTax": b["price"]["amount"], "amountAfterTax": b["price"]["amount"]} + for b in breakdown + ] + else: + prices = [{"amountBeforeTax": price, "amountAfterTax": price}] + + rooms.append({ + "roomId": room_id, + "rateId": rate_id, + "availability": available, + "prices": prices, + "min_stay_nights": None, + "currencyCode": "GBP", + }) + + return rooms diff --git a/backend/services/direct_profiles/guestline.py b/backend/services/direct_profiles/guestline.py new file mode 100644 index 0000000..bade8a4 --- /dev/null +++ b/backend/services/direct_profiles/guestline.py @@ -0,0 +1,49 @@ +import re +from datetime import date, timedelta +from .base import BaseProfile + +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Accept": "application/json", +} + + +class GuestlineProfile(BaseProfile): + name = "guestline" + label = "Guestline" + required_params = [ + {"key": "hotel_id", "label": "Hotel ID", "help": "e.g. THREEWAYS — from the booking URL ?hotel= parameter"}, + {"key": "collection_id", "label": "Collection ID", "help": "e.g. MT — the path segment before /availability in the booking URL"}, + ] + + @classmethod + def detect(cls, url: str) -> dict | None: + # Matches: https://booking.eu.guestline.app/MT/availability?hotel=THREEWAYS + m = re.search(r'booking\.(?:eu\.)?guestline\.app/([^/?\s]+)/availability\?hotel=([^&\s]+)', url) + if m: + return {"collection_id": m.group(1), "hotel_id": m.group(2)} + return None + + async def fetch_arrival_dates(self, client, params: dict) -> list[str]: + hotel_id = params["hotel_id"] + today = date.today() + url = f"https://booking.eu.guestline.app/api/availabilities/{hotel_id}/arrivals" + r = await client.get(url, params={ + "month": today.month, "year": today.year, + "adults": 2, "children": 0, "count": 12, + }, headers=HEADERS, timeout=20) + r.raise_for_status() + return [a["date"] for a in r.json().get("arrivals", [])] + + async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]: + hotel_id = params["hotel_id"] + collection_id = params.get("collection_id", "MT") + dep = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat() + url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}/enhanced" + r = await client.get(url, params={ + "arrival": arrival, "departure": dep, "adults": 2, "children": 0, + }, headers=HEADERS, timeout=20) + if r.status_code == 404: + return [] + r.raise_for_status() + return r.json().get("availabilities", {}).get("rooms", []) diff --git a/backend/services/direct_profiles/newbook_scrape.py b/backend/services/direct_profiles/newbook_scrape.py new file mode 100644 index 0000000..3b9420a --- /dev/null +++ b/backend/services/direct_profiles/newbook_scrape.py @@ -0,0 +1,235 @@ +import json +import re +from datetime import date, timedelta +from .base import BaseProfile + +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Accept": "*/*", + "X-Requested-With": "XMLHttpRequest", + "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", +} + +BASE_URL = "https://bookingseu.newbook.cloud" + + +def _base_params(slug: str, arrival: str, departure: str, nights: int) -> dict: + return { + "REMOTE_ADDR": "1.1.1.1", + "force_booking_channel_id": "", + "HTTP_REFERER": f"bookingseu.newbook.cloud/{slug}/index.php", + "discount_total_display": "0", + "force_category_id[]": "uK0ip@c7ty%8bQ#2i", + "force_category_type_id[]": "uK0ip@c7ty%8bQ#2i", + "no_billing_booking": "0", + "force_tariff_type_id[]": "uK0ip@c7ty%8bQ#2i", + "discount_id": "null", + "facebook_user_id": "", + "category_type_id": "", + "owner_occupied_booking_id": "", + "discount_code": "", + "booking_action": "", + "available_from": arrival, + "available_to": departure, + "nights": str(nights), + "adults": "2", + "children": "0", + "infants": "0", + "promo_code": "", + "language": "EN", + } + + +def _fmt_date(d: date) -> str: + """Format date as NewBook expects: 'Mon 4 Jul 2026'""" + return d.strftime("%a %-d %b %Y") + + +def _parse_chart_html(html: str) -> tuple[dict[str, float], dict[str, str], dict[str, str], list[dict]]: + """ + Parse an availability_chart_responsive HTML response. + + Returns: + counts: {cat_id: float} — room counts from category_sites_available JS var + cat_names: {cat_id: str} — friendly category names e.g. "Executive Double" + rate_names: {rate_id: str} — friendly tariff names e.g. "DIRECT B&B FLEX" + rooms: list of room/rate dicts compatible with base scraper format + """ + # Room counts from embedded JS + counts: dict[str, float] = {} + m = re.search(r'category_sites_available\s*=\s*(\{[^;]+\})', html) + if m: + try: + counts = {k: float(v) for k, v in json.loads(m.group(1)).items()} + except Exception: + pass + + cat_names: dict[str, str] = {} + rate_names: dict[str, str] = {} + rooms = [] + + # Split by category box: offset="{cat_id}" + cat_blocks = re.split(r']+class="[^"]*newbook_online_category_box[^"]*"[^>]+offset="(\d+)"', html) + # cat_blocks: [pre, cat_id, block, cat_id, block, ...] + i = 1 + while i < len(cat_blocks) - 1: + cat_id = cat_blocks[i] + block = cat_blocks[i + 1] + i += 2 + + avail = counts.get(cat_id, 0.0) + + # Category friendly name — from category_name attr on any book button in this block + # e.g. category_name='Standard' or category_name='Executive Double' + cn_m = re.search(r"category_name='([^']+)'", block) + if not cn_m: + # Fallback:

Name

+ cn_m = re.search(r'

[^<]*]*>([^<]+)', block) + if cn_m: + cat_names[cat_id] = cn_m.group(1).strip() + + # Split on tariff row boundaries + tariff_rows = re.split(r'class="[^"]*newbook_online_categories_tariff_type_rows[^"]*"', block) + + for row in tariff_rows[1:]: + # Rate label + name_m = re.search(r'newbook_online_categories_tariff_type_label[^>]*>(.*?)', row, re.DOTALL) + rate_label = re.sub(r'<[^>]+>', '', name_m.group(1)).strip() if name_m else "" + + # Price + price_m = re.search(r'newbook_online_from_price_text[^>]*>£([\d.]+)<', row) + price = float(price_m.group(1)) if price_m else None + + # Internal tariff type ID (stable across dates) + tid_m = re.search(r'tariff_type_id="(\d+)"', row) + rate_id = tid_m.group(1) if tid_m else rate_label + + # Store rate label for this rate_id + if rate_id and rate_label: + rate_names[rate_id] = rate_label + + # Min-stay: requires_date_change class + optional extend_nights attr + # If extend_nights present: min_stay = 1 + N; if absent: default to 2 + min_stay = None + if 'requires_date_change' in row: + en_m = re.search(r'extend_nights="(\d+)"', row) + min_stay = 1 + int(en_m.group(1)) if en_m else 2 + + if rate_label and price is not None: + rooms.append({ + "roomId": cat_id, + "rateId": rate_id, + "rateLabel": rate_label, + "availability": int(avail), + "prices": [{"amountBeforeTax": price, "amountAfterTax": price}], + "min_stay_nights": min_stay, + "currencyCode": "GBP", + }) + + return counts, cat_names, rate_names, rooms + + +class NewbookScrapeProfile(BaseProfile): + name = "newbook_scrape" + label = "NewBook (HTML scrape)" + required_params = [ + {"key": "slug", "label": "Property Slug", + "help": "The path segment in the booking URL, e.g. 'numberfour' from bookingseu.newbook.cloud/numberfour/"}, + ] + + @classmethod + def detect(cls, url: str) -> dict | None: + m = re.search(r'bookingseu\.newbook\.cloud/([^/?#\s]+)', url) + if m and m.group(1) not in ('index.php',): + return {"slug": m.group(1)} + return None + + async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]: + """ + Return ({cat_id: friendly_name}, {rate_id: friendly_name}) from a single chart call. + Used by discovery to pre-populate room_labels and rate_labels. + """ + slug = params["slug"] + today = date.today() + base = _base_params(slug, _fmt_date(today), _fmt_date(today + timedelta(days=1)), 1) + r = await client.post( + f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive", + data=base, headers=HEADERS, timeout=30, + ) + r.raise_for_status() + _, cat_names, rate_names, _ = _parse_chart_html(r.text) + return cat_names, rate_names + + async def fetch_arrival_dates(self, client, params: dict) -> list[str]: + """ + Use the calendar endpoint to collect all available arrival dates + across all room types. One call per room type, union of available dates. + We first do a chart call to discover category IDs, then calendar per category. + """ + slug = params["slug"] + today = date.today() + arrival_str = _fmt_date(today) + departure_str = _fmt_date(today + timedelta(days=1)) + + # Step 1: chart call to discover category IDs and names + base = _base_params(slug, arrival_str, departure_str, 1) + r = await client.post( + f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive", + data=base, headers=HEADERS, timeout=30, + ) + r.raise_for_status() + html = r.text + + cat_ids = re.findall(r'class="[^"]*newbook_online_category_box[^"]*"[^>]+offset="(\d+)"', html) + if not cat_ids: + return [] + + # Step 2: calendar call per category, collect available dates + available_dates: set[str] = set() + more_tariffs = {f"more_tariffs_{cid}": "1" for cid in cat_ids} + + for cat_id in cat_ids: + cal_params = { + **base, + **more_tariffs, + "query": "newbook_calendar_initialise", + "calendar_category_id": cat_id, + } + try: + cr = await client.post( + f"{BASE_URL}/{slug}/api.php?newbook_api_action=data", + data=cal_params, headers=HEADERS, timeout=30, + ) + cr.raise_for_status() + cal_data = cr.json() + cal_html = cal_data.get("calendar_display", "") + for dm in re.finditer(r'class="day available[^"]*"\s+data-date="(\d{4}-\d{2}-\d{2})"', cal_html): + available_dates.add(dm.group(1)) + except Exception: + pass + + return sorted(d for d in available_dates if d >= today.isoformat()) + + async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]: + """ + POST to availability_chart_responsive for a specific date. + Returns list of room/rate dicts compatible with the base scraper format. + Also injects min_stay_nights onto each row. + """ + slug = params["slug"] + arr = date.fromisoformat(arrival) + dep = arr + timedelta(days=nights) + arr_str = _fmt_date(arr) + dep_str = _fmt_date(dep) + + body = _base_params(slug, arr_str, dep_str, nights) + r = await client.post( + f"{BASE_URL}/{slug}/api.php?newbook_api_action=availability_chart_responsive", + data=body, headers=HEADERS, timeout=30, + ) + if r.status_code == 404: + return [] + r.raise_for_status() + + _, _, _, rooms = _parse_chart_html(r.text) + return rooms diff --git a/backend/services/direct_profiles/travelclick.py b/backend/services/direct_profiles/travelclick.py new file mode 100644 index 0000000..093f7af --- /dev/null +++ b/backend/services/direct_profiles/travelclick.py @@ -0,0 +1,166 @@ +import re +import httpx +from datetime import date, timedelta +from .base import BaseProfile + +API_BASE = "https://api.travelclick.com" +TOKEN_URL = f"{API_BASE}/oauth/token-referer?grant_type=client_credentials" + +HEADERS_BASE = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Accept": "application/json", + "Content-Type": "application/json", +} + + +async def _get_token(client, referer: str) -> str: + r = await client.post(TOKEN_URL, headers={**HEADERS_BASE, "Referer": referer}, timeout=20) + r.raise_for_status() + return r.json()["access_token"] + + +def _auth_headers(token: str, referer: str) -> dict: + return {**HEADERS_BASE, "Authorization": f"Bearer {token}", "Referer": referer} + + +class TravelClickProfile(BaseProfile): + name = "travelclick" + label = "TravelClick / Amadeus" + required_params = [ + {"key": "hotel_code", "label": "Hotel Code", + "help": "Numeric hotel ID, e.g. 77346 — visible in the booking engine network requests"}, + {"key": "booking_url", "label": "Booking URL Base", + "help": "e.g. https://reservations.bespokehotels.com/noelarmshotel/book/dates-of-stay"}, + ] + + @classmethod + def detect(cls, url: str) -> dict | None: + m = re.search(r'(https://reservations\.bespokehotels\.com/[^/]+/book/[^?#\s]+)', url) + if not m: + return None + booking_url = m.group(1) + # hotel_code is in inline JS as bookingEngineHotelId: '77346' on the booking page + hotel_code = "" + try: + r = httpx.get(booking_url, timeout=10, follow_redirects=True, + headers={"User-Agent": "Mozilla/5.0"}) + hm = re.search(r'bookingEngineHotelId\s*:\s*[\'"](\d+)[\'"]', r.text) + if hm: + hotel_code = hm.group(1) + except Exception: + pass + return {"booking_url": booking_url, "hotel_code": hotel_code} + + async def fetch_category_names(self, client, params: dict) -> tuple[dict[str, str], dict[str, str]]: + hotel_code = params["hotel_code"] + referer = params["booking_url"] + token = await _get_token(client, referer) + r = await client.get( + f"{API_BASE}/be5-entity/v2/hotels/{hotel_code}/content", + params={"include": "roomtypes,rateplans", "lang": "EN_US"}, + headers=_auth_headers(token, referer), + timeout=30, + ) + r.raise_for_status() + data = r.json() + # roomtypes[].roomTypeId (numeric, used as roomId in avail) -> roomTypeName + room_labels = { + str(rt["roomTypeId"]): rt.get("roomTypeName", str(rt["roomTypeId"])) + for rt in data.get("roomtypes", []) + } + # ratePlans[].rateplanId (numeric, used as rateId in avail) -> rateplanName + rate_labels = { + str(rp["rateplanId"]): rp.get("rateplanName", str(rp["rateplanId"])) + for rp in data.get("ratePlans", []) + } + return room_labels, rate_labels + + async def fetch_arrival_dates(self, client, params: dict) -> list[str]: + hotel_code = params["hotel_code"] + referer = params["booking_url"] + token = await _get_token(client, referer) + today = date.today() + end = today + timedelta(days=365) + body = { + "hotelCode": int(hotel_code), + "currency": "GBP", + "lang": "EN_US", + "dateIn": today.isoformat(), + "dateOut": end.isoformat(), + "multiRoomOccupancy": [{"adults": 2, "infant": 0, "children": 0}], + "bookerIdentifier": "", + "partnerIdentifier": "", + } + r = await client.post( + f"{API_BASE}/be5-shop/v1/hotel/{hotel_code}/basicavail/multi-room", + json=body, + headers=_auth_headers(token, referer), + timeout=30, + ) + r.raise_for_status() + data = r.json() + return [ + d["date"] + for d in data.get("dates", []) + if d.get("isAvailable") and d["date"] >= today.isoformat() + ] + + async def fetch_night_rates(self, client, params: dict, arrival: str, nights: int = 1) -> list[dict]: + hotel_code = params["hotel_code"] + referer = params["booking_url"] + token = await _get_token(client, referer) + departure = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat() + body = { + "roomStay": { + "startDate": arrival, + "endDate": departure, + "guestCount": {"adults": 2, "infants": 0, "children": {"ages": None, "count": 0}}, + "roomQuantity": 1, + "productSearchCriteria": { + "sortingPreference": "SORT_BY_ORDER", + "includeUnavailable": False, + }, + }, + "languageCode": "EN_US", + "disableLocaitonSharing": False, + "currencyCode": "GBP", + "tpaExtension": [], + "includeMemberRate": True, + "includeNightlyRates": True, + } + r = await client.post( + f"{API_BASE}/be5-shop/v2/hotels/{hotel_code}/avail", + json=body, + headers=_auth_headers(token, referer), + timeout=30, + ) + if r.status_code == 404: + return [] + r.raise_for_status() + + data = r.json() + # Response: roomStays[0].roomtypes[].products[] (both Regular rates and Packages) + room_stay = (data.get("roomStays") or [{}])[0] + rooms = [] + for rt in room_stay.get("roomtypes", []): + room_id = str(rt["roomtypeId"]) + for product in rt.get("products", []): + rate_id = str(product["productId"]) + nightly = product.get("nightlyRates", []) + if not nightly: + continue + avail = nightly[0].get("inventoryCount", 0) + prices = [ + {"amountBeforeTax": n["amountBeforeTax"], "amountAfterTax": n.get("amountTotal", n["amountBeforeTax"])} + for n in nightly + ] + rooms.append({ + "roomId": room_id, + "rateId": rate_id, + "availability": avail, + "prices": prices, + "min_stay_nights": None, # min-stay comes from basicavail per date + "currencyCode": "GBP", + }) + + return rooms diff --git a/backend/services/direct_scraper.py b/backend/services/direct_scraper.py new file mode 100644 index 0000000..aba16e2 --- /dev/null +++ b/backend/services/direct_scraper.py @@ -0,0 +1,295 @@ +""" +Direct booking engine scraper — adapted from guestline-monitor/app/scraper.py. +Replaces SQLite per-hotel DBs with PostgreSQL via SyncSessionLocal. +Logic (min-stay detection, discovery date sampling) is unchanged from the original. +""" +import asyncio +import logging +from datetime import datetime, timezone, date, timedelta + +import httpx +from sqlalchemy import text + +from database import SyncSessionLocal +from services.direct_profiles import get_profile + +log = logging.getLogger(__name__) + +REQUEST_DELAY = 10.0 +DISCOVERY_DELAY = 2.0 + +# Track running discovery scrapes: hotel_id -> status dict +_discovery_status: dict[int, dict] = {} + + +def _discovery_dates() -> list[str]: + """42 spread dates: one per DOW over 6 months, 6 weeks apart.""" + dates = [] + today = date.today() + for week_offset in range(6): + base = today + timedelta(weeks=week_offset * 4) + for dow in range(7): + days_ahead = (dow - base.weekday()) % 7 + d = base + timedelta(days=days_ahead + 7) + iso = d.isoformat() + if iso not in dates: + dates.append(iso) + return sorted(dates) + + +async def run_discovery(hotel_id: int, profile_name: str, params: dict): + """Sample ~42 spread dates to find all room/rate type IDs for a competitor hotel.""" + _discovery_status[hotel_id] = { + "state": "running", "done": 0, "total": 0, + "found_rooms": [], "found_rates": [], + } + profile = get_profile(profile_name) + dates = _discovery_dates() + _discovery_status[hotel_id]["total"] = len(dates) + + found_rooms: set[str] = set() + found_rates: set[str] = set() + + async with httpx.AsyncClient() as client: + for i, arrival in enumerate(dates): + await asyncio.sleep(DISCOVERY_DELAY) + try: + rooms = await profile.fetch_night_rates(client, params, arrival) + for room in rooms: + found_rooms.add(room["roomId"]) + found_rates.add(room["rateId"]) + except Exception as e: + log.warning(f"Discovery hotel {hotel_id} {arrival}: {e}") + _discovery_status[hotel_id]["done"] = i + 1 + _discovery_status[hotel_id]["found_rooms"] = sorted(found_rooms) + _discovery_status[hotel_id]["found_rates"] = sorted(found_rates) + + # Fetch friendly names if the profile supports it + if hasattr(profile, "fetch_category_names"): + try: + cat_names, rate_names = await profile.fetch_category_names(client, params) + _discovery_status[hotel_id]["room_labels"] = cat_names + _discovery_status[hotel_id]["rate_labels"] = rate_names + # Merge into DB (existing user labels win) + db = SyncSessionLocal() + try: + row = db.execute( + text("SELECT room_labels, rate_labels FROM direct_competitor_hotels WHERE id = :id"), + {"id": hotel_id} + ).mappings().fetchone() + if row: + import json + existing_rooms = row["room_labels"] or {} + existing_rates = row["rate_labels"] or {} + merged_rooms = {**cat_names, **existing_rooms} + merged_rates = {**rate_names, **existing_rates} + db.execute( + text("""UPDATE direct_competitor_hotels + SET room_labels = :rl, rate_labels = :ratel + WHERE id = :id"""), + {"rl": json.dumps(merged_rooms), "ratel": json.dumps(merged_rates), "id": hotel_id} + ) + db.commit() + finally: + db.close() + except Exception as e: + log.warning(f"Discovery hotel {hotel_id}: could not fetch category names: {e}") + + _discovery_status[hotel_id]["state"] = "complete" + log.info(f"Discovery complete hotel {hotel_id}: {len(found_rooms)} rooms, {len(found_rates)} rates") + + +def get_discovery_status(hotel_id: int) -> dict: + return _discovery_status.get(hotel_id, {"state": "idle"}) + + +def run_scrape(hotel_id: int, profile_name: str, params: dict): + """Full scrape run for one configured hotel. Writes to direct_rates + direct_scrape_runs.""" + scraped_at = datetime.now(timezone.utc) + log.info(f"Direct scrape started for hotel {hotel_id}") + profile = get_profile(profile_name) + + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at)) + finally: + loop.close() + + db = SyncSessionLocal() + try: + db.execute( + text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"), + {"ts": scraped_at, "id": hotel_id} + ) + db.commit() + finally: + db.close() + + +async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime): + async with httpx.AsyncClient() as client: + try: + arrival_dates = await profile.fetch_arrival_dates(client, params) + except Exception as e: + log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}") + return + + log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates") + + db = SyncSessionLocal() + try: + result = db.execute( + text("INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found) VALUES (:hid, :ts, :df) RETURNING id"), + {"hid": hotel_id, "ts": scraped_at, "df": len(arrival_dates)} + ) + run_id = result.fetchone()[0] + db.commit() + + prev_dates = {r[0].isoformat() for r in db.execute( + text("SELECT DISTINCT stay_date FROM direct_rates WHERE hotel_id = :hid AND stay_date >= :today"), + {"hid": hotel_id, "today": date.today()} + ).fetchall()} + finally: + db.close() + + arrival_set = set(arrival_dates) + missing_dates = sorted(prev_dates - arrival_set) + if missing_dates: + log.info(f"Hotel {hotel_id}: {len(missing_dates)} dates absent, checking min-stay") + + # Min-stay check for missing dates + for fd in missing_dates: + fd_date = date.fromisoformat(fd) + windows = [ + (fd_date - timedelta(days=1), fd_date + timedelta(days=1)), + (fd_date, fd_date + timedelta(days=2)), + ] + min_stay_rooms = None + min_stay_other_night = None + + await asyncio.sleep(REQUEST_DELAY) + for win_start, win_end in windows: + try: + rooms_2n = await profile.fetch_night_rates(client, params, win_start.isoformat(), nights=2) + if rooms_2n: + min_stay_rooms = rooms_2n + companion = win_start if win_start.isoformat() != fd else (win_start + timedelta(days=1)) + min_stay_other_night = companion.isoformat() + log.info(f" Hotel {hotel_id} {fd}: min-stay detected") + break + except Exception as e: + log.warning(f" Hotel {hotel_id} {fd}: min-stay check {win_start}: {e}") + + db = SyncSessionLocal() + try: + last_rows = db.execute( + text("""SELECT DISTINCT ON (room_id, rate_id) + room_id, rate_id, availability, price_excl, price_incl, currency + FROM direct_rates + WHERE hotel_id = :hid AND stay_date = :fd + ORDER BY room_id, rate_id, scraped_at DESC"""), + {"hid": hotel_id, "fd": fd} + ).mappings().fetchall() + + insert_rows = [] + for r in last_rows: + if min_stay_rooms is not None: + match = next( + (m for m in min_stay_rooms + if m["roomId"] == r["room_id"] and m["rateId"] == r["rate_id"] + and m.get("prices")), + None + ) + if match: + prices = match["prices"] + win_start_used = windows[0][0] if min_stay_other_night == windows[0][0].isoformat() else windows[1][0] + idx = 1 if win_start_used.isoformat() != fd else 0 + if len(prices) > idx: + p = prices[idx] + insert_rows.append({ + "hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd, + "room_id": r["room_id"], "rate_id": r["rate_id"], + "avail": match.get("availability", r["availability"]), + "pe": p["amountBeforeTax"], "pi": p["amountAfterTax"], + "cur": match.get("currencyCode", r["currency"]), "ms": 2 + }) + continue + insert_rows.append({ + "hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd, + "room_id": r["room_id"], "rate_id": r["rate_id"], + "avail": r["availability"], "pe": r["price_excl"], "pi": r["price_incl"], + "cur": r["currency"], "ms": 2 + }) + else: + insert_rows.append({ + "hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd, + "room_id": r["room_id"], "rate_id": r["rate_id"], + "avail": 0, "pe": r["price_excl"], "pi": r["price_incl"], + "cur": r["currency"], "ms": None + }) + + if insert_rows: + db.execute( + text("""INSERT INTO direct_rates + (hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id, + availability, price_excl, price_incl, currency, min_stay_nights) + VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id, + :avail, :pe, :pi, :cur, :ms)"""), + insert_rows + ) + db.commit() + status = "min-stay(2N)" if min_stay_rooms else "fully-booked" + log.info(f" Hotel {hotel_id} {fd}: recorded as {status}") + finally: + db.close() + + # Scrape all bookable dates + rows_saved = 0 + for i, arrival in enumerate(arrival_dates): + await asyncio.sleep(REQUEST_DELAY) + try: + rooms = await profile.fetch_night_rates(client, params, arrival) + if not rooms: + continue + insert_rows = [ + { + "hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": arrival, + "room_id": room["roomId"], "rate_id": room["rateId"], + "avail": room["availability"], + "pe": room["prices"][0]["amountBeforeTax"], + "pi": room["prices"][0]["amountAfterTax"], + "cur": room.get("currencyCode", "GBP"), + "ms": room.get("min_stay_nights") + } + for room in rooms if room.get("prices") + ] + if insert_rows: + db = SyncSessionLocal() + try: + db.execute( + text("""INSERT INTO direct_rates + (hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id, + availability, price_excl, price_incl, currency, min_stay_nights) + VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id, + :avail, :pe, :pi, :cur, :ms)"""), + insert_rows + ) + db.commit() + finally: + db.close() + rows_saved += len(insert_rows) + log.info(f" Hotel {hotel_id} {arrival}: {len(rooms)} combos [{i+1}/{len(arrival_dates)}]") + except Exception as e: + log.error(f" Hotel {hotel_id} {arrival}: ERROR - {e}") + + db = SyncSessionLocal() + try: + db.execute( + text("UPDATE direct_scrape_runs SET rows_saved = :rs WHERE id = :id"), + {"rs": rows_saved, "id": run_id} + ) + db.commit() + finally: + db.close() + + log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved") diff --git a/backend/services/newbook_rates_client.py b/backend/services/newbook_rates_client.py new file mode 100644 index 0000000..64a1e2d --- /dev/null +++ b/backend/services/newbook_rates_client.py @@ -0,0 +1,863 @@ +""" +Newbook Rates Client + +Fetches current rack rates from Newbook API for revenue forecasting. +Uses the bookings_availability_pricing endpoint to simulate booking requests. + +This client is READ-ONLY - it only queries available rates, never creates bookings. +""" +import os +import httpx +import asyncio +import logging +from datetime import date, timedelta +from decimal import Decimal +from typing import Optional, List, Dict + +logger = logging.getLogger(__name__) + + +class NewbookRatesError(Exception): + """Custom exception for Newbook rates API errors""" + pass + + +class NewbookRatesClient: + """ + Async client for fetching current rates from Newbook API. + + Uses bookings_availability_pricing endpoint which simulates a booking request. + Handles minimum stay restrictions by extending the stay period when needed. + + Rate limiting: ~100 requests/min, using 0.75s delay between requests + """ + + BASE_URL = "https://api.newbook.cloud/rest" + + def __init__(self, api_key: str = None, username: str = None, password: str = None, + region: str = None, vat_rate: Decimal = Decimal('0.20')): + self.api_key = api_key or os.getenv("NEWBOOK_API_KEY") + self.username = username or os.getenv("NEWBOOK_USERNAME") + self.password = password or os.getenv("NEWBOOK_PASSWORD") + self.region = region or os.getenv("NEWBOOK_REGION") + self.vat_rate = vat_rate + + if not all([self.api_key, self.username, self.password, self.region]): + logger.warning("Newbook credentials not fully configured") + + def _get_url(self, endpoint: str) -> str: + """Get full URL for an endpoint""" + return f"{self.BASE_URL}/{endpoint}" + + @classmethod + async def from_db(cls, db): + """ + Create client with credentials from the central Settings service + (stack-wide NewBook config), falling back to the app-local + system_config table. VAT rate stays app-local either way. + """ + from sqlalchemy import text + from services.central_settings import get_newbook_credentials + + result = await db.execute( + text("SELECT config_key, config_value FROM system_config WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')") + ) + rows = result.fetchall() + config = {row.config_key: row.config_value for row in rows} + + vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20')) + + central = await get_newbook_credentials() + if central: + return cls(**central, vat_rate=vat_rate) + + return cls( + api_key=config.get('newbook_api_key'), + username=config.get('newbook_username'), + password=config.get('newbook_password'), + region=config.get('newbook_region'), + vat_rate=vat_rate + ) + + async def __aenter__(self): + self.client = httpx.AsyncClient(timeout=60.0) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.client.aclose() + + def _get_auth_payload(self) -> dict: + """Get base authentication payload""" + return { + "api_key": self.api_key, + "region": self.region + } + + async def get_category_rates( + self, + category_id: str, + from_date: date, + to_date: date, + guests_adults: int = 2, + guests_children: int = 0 + ) -> List[Dict]: + """ + Fetch current rates for a category over a date range. + + Uses daily=true to get per-night rates. Handles minimum stay + restrictions by extending the period when needed. + + Args: + category_id: Newbook category ID + from_date: Start date for rates + to_date: End date for rates (inclusive) + guests_adults: Number of adult guests (default 2) + guests_children: Number of child guests (default 0) + + Returns: + List of dicts with {date, gross_rate, net_rate} + """ + rates = [] + current_date = from_date + + while current_date <= to_date: + try: + # Fetch rates for up to 7 days at a time to optimize API calls + batch_end = min(current_date + timedelta(days=6), to_date) + batch_rates = await self._fetch_rates_batch( + category_id, current_date, batch_end, guests_adults, guests_children + ) + rates.extend(batch_rates) + + # Move to next batch + current_date = batch_end + timedelta(days=1) + + except Exception as e: + logger.error(f"Failed to fetch rates for category {category_id} starting {current_date}: {e}") + # Skip this batch and continue + current_date = current_date + timedelta(days=7) + + # Rate limiting - ALWAYS wait 1.5s between requests, even after errors + await asyncio.sleep(1.5) + + return rates + + async def get_single_night_rates( + self, + category_id: str, + from_date: date, + to_date: date, + guests_adults: int = 2, + guests_children: int = 0 + ) -> List[Dict]: + """ + Fetch rates with single-night queries for accurate per-day tariff availability. + + Unlike get_category_rates which batches, this queries each date individually + as a 1-night stay. This gives accurate tariff_success per night, catching + issues like Valentine's Day blocking only that night, not a whole week. + + Much slower but necessary for accurate bookability data. + + Args: + category_id: Newbook category ID + from_date: Start date for rates + to_date: End date for rates (inclusive) + guests_adults: Number of adult guests (default 2) + guests_children: Number of child guests (default 0) + + Returns: + List of dicts with {date, gross_rate, net_rate, tariffs_data} + """ + rates = [] + current_date = from_date + + while current_date <= to_date: + try: + # Single-night query for accurate tariff availability + batch_rates = await self._fetch_rates_batch( + category_id, current_date, current_date, guests_adults, guests_children + ) + rates.extend(batch_rates) + + except Exception as e: + logger.warning(f"Failed to fetch single-night rate for {category_id} on {current_date}: {e}") + # Continue with next date + + current_date += timedelta(days=1) + + # Rate limiting - wait between each single-night query + await asyncio.sleep(1.0) + + return rates + + async def fetch_single_date_all_categories( + self, + for_date: date, + guests_adults: int = 2, + guests_children: int = 0 + ) -> Dict[str, List[Dict]]: + """ + Fetch single-night rates for ALL categories for one date. + + Returns: + Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]} + """ + return await self._fetch_all_categories_batch( + for_date, guests_adults, guests_children + ) + + async def fetch_multi_night_for_date( + self, + for_date: date, + nights: int, + guests_adults: int = 2, + guests_children: int = 0 + ) -> Dict[str, Dict[str, bool]]: + """ + Fetch multi-night availability for ALL categories for one date. + + Returns: + Dict of {category_id: {tariff_name: available}} + """ + return await self._fetch_all_categories_multi_night( + for_date, nights, guests_adults, guests_children + ) + + async def get_all_categories_single_night_rates( + self, + from_date: date, + to_date: date, + guests_adults: int = 2, + guests_children: int = 0 + ) -> Dict[str, List[Dict]]: + """ + Fetch rates for ALL categories with single-night queries. + + More efficient than get_single_night_rates - omits category_id to get + all categories in a single API call per date. This reduces API calls + from (categories × days) to just (days). + + Args: + from_date: Start date for rates + to_date: End date for rates (inclusive) + guests_adults: Number of adult guests (default 2) + guests_children: Number of child guests (default 0) + + Returns: + Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}, ...]} + """ + all_rates: Dict[str, List[Dict]] = {} + current_date = from_date + total_days = (to_date - from_date).days + 1 + day_count = 0 + + while current_date <= to_date: + day_count += 1 + try: + # Single-night query WITHOUT category_id - returns ALL categories + category_rates = await self._fetch_all_categories_batch( + current_date, guests_adults, guests_children + ) + + # Merge into all_rates dict + for cat_id, rates in category_rates.items(): + if cat_id not in all_rates: + all_rates[cat_id] = [] + all_rates[cat_id].extend(rates) + + logger.info(f"Fetched {current_date} ({day_count}/{total_days}) - {len(category_rates)} categories") + + except Exception as e: + logger.warning(f"Failed to fetch rates for {current_date}: {e}") + # Continue with next date + + current_date += timedelta(days=1) + + # Rate limiting - wait between each query + await asyncio.sleep(1.0) + + return all_rates + + async def _fetch_all_categories_batch( + self, + for_date: date, + guests_adults: int, + guests_children: int, + retry_count: int = 0 + ) -> Dict[str, List[Dict]]: + """ + Fetch rates for ALL categories for a single date. + + Omits category_id from request - Newbook returns all available categories. + + Returns: + Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]} + """ + # Single-night query + period_from = f"{for_date.isoformat()} 14:00:00" + period_to = f"{(for_date + timedelta(days=1)).isoformat()} 10:00:00" + + payload = self._get_auth_payload() + payload.update({ + "period_from": period_from, + "period_to": period_to, + "adults": guests_adults, + "children": guests_children, + "infants": 0, + "daily_mode": "true" + # NO category_id - returns all categories + }) + + response = await self.client.post( + self._get_url("bookings_availability_pricing"), + json=payload, + auth=(self.username, self.password) + ) + + # Handle rate limiting with exponential backoff + if response.status_code == 429: + if retry_count < 3: + wait_time = 60 * (retry_count + 1) + logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3") + await asyncio.sleep(wait_time) + return await self._fetch_all_categories_batch( + for_date, guests_adults, guests_children, retry_count + 1 + ) + else: + raise NewbookRatesError(f"Rate limited after 3 retries") + + if response.status_code != 200: + raise NewbookRatesError(f"API error {response.status_code}: {response.text}") + + data = response.json() + + if not data.get("success"): + raise NewbookRatesError(f"API returned failure: {data.get('message')}") + + # Parse all categories from response + return self._parse_all_categories_tariffs(data, for_date) + + async def _fetch_all_categories_multi_night( + self, + for_date: date, + nights: int, + guests_adults: int = 2, + guests_children: int = 0, + retry_count: int = 0 + ) -> Dict[str, Dict[str, bool]]: + """ + Fetch multi-night availability for ALL categories for a specific date. + + Used to verify that rates with min_stay requirements are actually bookable. + + Args: + for_date: Check-in date + nights: Number of nights to query (e.g., 2 for min_stay=2) + guests_adults: Number of adult guests + guests_children: Number of child guests + + Returns: + Dict of {category_id: {tariff_name: available}} + """ + # Multi-night query + period_from = f"{for_date.isoformat()} 14:00:00" + period_to = f"{(for_date + timedelta(days=nights)).isoformat()} 10:00:00" + + payload = self._get_auth_payload() + payload.update({ + "period_from": period_from, + "period_to": period_to, + "adults": guests_adults, + "children": guests_children, + "infants": 0, + "daily_mode": "true" + }) + + response = await self.client.post( + self._get_url("bookings_availability_pricing"), + json=payload, + auth=(self.username, self.password) + ) + + # Handle rate limiting + if response.status_code == 429: + if retry_count < 3: + wait_time = 60 * (retry_count + 1) + logger.warning(f"Rate limited (multi-night), waiting {wait_time}s") + await asyncio.sleep(wait_time) + return await self._fetch_all_categories_multi_night( + for_date, nights, guests_adults, guests_children, retry_count + 1 + ) + else: + raise NewbookRatesError(f"Rate limited after 3 retries") + + if response.status_code != 200: + raise NewbookRatesError(f"API error {response.status_code}: {response.text}") + + data = response.json() + + if not data.get("success"): + raise NewbookRatesError(f"API returned failure: {data.get('message')}") + + # Parse availability by tariff name for each category + results: Dict[str, Dict[str, bool]] = {} + + if not isinstance(data.get("data"), dict): + return results + + for key, cat_data in data["data"].items(): + if not (key.isdigit() or str(key).isnumeric()): + continue + if not isinstance(cat_data, dict): + continue + + category_id = str(key) + tariffs_available = cat_data.get("tariffs_available", []) + + results[category_id] = {} + for tariff in tariffs_available: + tariff_name = tariff.get("tariff_name", "") + tariff_label = tariff.get("tariff_label", "") + # Check tariff_success (API returns string "true"/"false") + tariff_success = str(tariff.get("tariff_success", False)).lower() in ("true", "1") + # Available if API says success, OR if rates are quoted and no restriction message + is_available = tariff_success or ( + bool(tariff.get("tariffs_quoted")) and not tariff.get("tariff_message") + ) + # Store under both tariff_name and tariff_label for flexible matching + results[category_id][tariff_name] = is_available + if tariff_label and tariff_label != tariff_name: + results[category_id][tariff_label] = is_available + + return results + + async def get_multi_night_availability( + self, + dates_by_nights: Dict[int, List[date]], + guests_adults: int = 2, + guests_children: int = 0 + ) -> Dict[date, Dict[str, Dict[str, bool]]]: + """ + Fetch multi-night availability for specific dates grouped by stay length. + + Checks if a tariff is available when booking N nights starting from each date. + + Args: + dates_by_nights: Dict of {nights: [dates]} e.g., {2: [date1, date2], 3: [date3]} + guests_adults: Number of adult guests + guests_children: Number of child guests + + Returns: + Dict of {date: {category_id: {tariff_name: available}}} + """ + results: Dict[date, Dict[str, Dict[str, bool]]] = {} + + total_queries = sum(len(dates) for dates in dates_by_nights.values()) + query_count = 0 + + for nights, dates in dates_by_nights.items(): + for query_date in dates: + query_count += 1 + + try: + result = await self._fetch_all_categories_multi_night( + query_date, nights, guests_adults, guests_children + ) + results[query_date] = result + logger.info(f"Multi-night check {query_count}/{total_queries}: {query_date} ({nights} nights)") + except Exception as e: + logger.warning(f"Failed multi-night check for {query_date}: {e}") + + # Rate limiting + await asyncio.sleep(1.0) + + return results + + async def _fetch_rates_batch( + self, + category_id: str, + from_date: date, + to_date: date, + guests_adults: int, + guests_children: int, + retry_count: int = 0 + ) -> List[Dict]: + """ + Fetch rates for a batch of dates (up to 7 days). + + Handles minimum stay restrictions by extending the period and + extracting only the dates we need. + + Returns: + List of dicts with {date, gross_rate, net_rate} + """ + # Format dates with times (check-in 14:00, check-out 10:00) + period_from = f"{from_date.isoformat()} 14:00:00" + period_to = f"{(to_date + timedelta(days=1)).isoformat()} 10:00:00" + + payload = self._get_auth_payload() + payload.update({ + "period_from": period_from, + "period_to": period_to, + "adults": guests_adults, + "children": guests_children, + "infants": 0, + "category_id": category_id, + "daily_mode": "true" # Get per-night breakdown + }) + + response = await self.client.post( + self._get_url("bookings_availability_pricing"), + json=payload, + auth=(self.username, self.password) + ) + + # Handle rate limiting with exponential backoff + if response.status_code == 429: + if retry_count < 3: + wait_time = 60 * (retry_count + 1) # 60s, 120s, 180s + logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3") + await asyncio.sleep(wait_time) + return await self._fetch_rates_batch( + category_id, from_date, to_date, guests_adults, guests_children, retry_count + 1 + ) + else: + raise NewbookRatesError(f"Rate limited after 3 retries") + + if response.status_code != 200: + raise NewbookRatesError(f"API error {response.status_code}: {response.text}") + + data = response.json() + + if not data.get("success"): + # Check if minimum stay restriction + categories = data.get("data", {}).get("categories", []) + if categories: + cat = categories[0] if isinstance(categories, list) else categories.get(category_id, {}) + min_periods = cat.get("minimum_periods", 1) + + if min_periods > 1: + # Extend the stay to meet minimum and retry + extended_to = from_date + timedelta(days=min_periods) + logger.info(f"Minimum stay {min_periods} nights for category {category_id}, extending to {extended_to}") + return await self._fetch_rates_with_min_stay( + category_id, from_date, to_date, extended_to, + guests_adults, guests_children + ) + + raise NewbookRatesError(f"API returned failure: {data.get('message')}") + + # Parse tariffs_quoted from response + return self._parse_tariffs(data, from_date, to_date) + + async def _fetch_rates_with_min_stay( + self, + category_id: str, + from_date: date, + to_date: date, + extended_to: date, + guests_adults: int, + guests_children: int, + retry_count: int = 0 + ) -> List[Dict]: + """ + Fetch rates with extended period for minimum stay requirement. + + Args: + category_id: Newbook category ID + from_date: Original start date + to_date: Original end date (dates we want) + extended_to: Extended end date to meet minimum stay + guests_adults: Number of adults + guests_children: Number of children + + Returns: + List of rates for the original date range only + """ + period_from = f"{from_date.isoformat()} 14:00:00" + period_to = f"{(extended_to + timedelta(days=1)).isoformat()} 10:00:00" + + payload = self._get_auth_payload() + payload.update({ + "period_from": period_from, + "period_to": period_to, + "adults": guests_adults, + "children": guests_children, + "infants": 0, + "category_id": category_id, + "daily_mode": "true" + }) + + response = await self.client.post( + self._get_url("bookings_availability_pricing"), + json=payload, + auth=(self.username, self.password) + ) + + # Handle rate limiting with exponential backoff + if response.status_code == 429: + if retry_count < 3: + wait_time = 60 * (retry_count + 1) + logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry") + await asyncio.sleep(wait_time) + return await self._fetch_rates_with_min_stay( + category_id, from_date, to_date, extended_to, guests_adults, guests_children, retry_count + 1 + ) + else: + raise NewbookRatesError(f"Rate limited after 3 retries") + + if response.status_code != 200: + raise NewbookRatesError(f"API error {response.status_code}: {response.text}") + + data = response.json() + + if not data.get("success"): + raise NewbookRatesError(f"API returned failure even with extended stay: {data.get('message')}") + + # Parse tariffs but only return dates in our original range + return self._parse_tariffs(data, from_date, to_date) + + def _parse_tariffs(self, data: dict, from_date: date, to_date: date) -> List[Dict]: + """ + Parse tariffs from API response. + + With daily_mode=true, the API returns tariffs_quoted as a dict keyed by date. + Falls back to tariffs_available average if tariffs_quoted not available. + + Args: + data: Full API response + from_date: Start date to include + to_date: End date to include + + Returns: + List of dicts with {date, gross_rate, net_rate, tariffs_data} + tariffs_data contains all available tariff options for rate report + """ + rates = [] + tariffs_quoted = {} + fallback_rate = None + inventory_items = [] + all_tariffs_available = [] # Store all tariff options for reporting + + # Find tariffs data in the response + if isinstance(data.get("data"), dict): + for key in data["data"].keys(): + # Category IDs are numeric strings + if key.isdigit() or key.isnumeric(): + cat_data = data["data"][key] + if isinstance(cat_data, dict): + tariffs_available = cat_data.get("tariffs_available", []) + all_tariffs_available = tariffs_available # Capture all options + if tariffs_available: + first_tariff = tariffs_available[0] + # tariffs_quoted is a dict keyed by date string + tariffs_quoted = first_tariff.get("tariffs_quoted", {}) + # inventory_items are at tariff level (total for whole stay) + inventory_items = first_tariff.get("inventory_items", []) + # Fallback average rate + fallback_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0)) + break + + # If we have per-night tariffs_quoted dict, parse it + if isinstance(tariffs_quoted, dict) and tariffs_quoted: + num_nights = len(tariffs_quoted) + + # Calculate per-night inventory item amount for items already included in tariff + included_inventory_per_night = Decimal('0') + for item in inventory_items: + already_included = item.get('amount_already_included_in_tariff_total', '') + if str(already_included).lower() == 'true': + total_amount = Decimal(str(item.get('amount', 0) or 0)) + included_inventory_per_night += total_amount / num_nights + + for date_str, tariff in tariffs_quoted.items(): + try: + stay_date = date.fromisoformat(date_str) + except ValueError: + continue + + # Only include dates in our range + if stay_date < from_date or stay_date > to_date: + continue + + gross_rate = Decimal(str(tariff.get('amount', 0) or 0)) + # Net = (gross - included_inventory_per_night) / (1 + VAT) + gross_after_inventory = gross_rate - included_inventory_per_night + net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01')) + + # Build tariffs_data with day-specific rates + tariffs_data = self._build_tariffs_summary(all_tariffs_available, stay_date) + + rates.append({ + 'date': stay_date, + 'gross_rate': float(gross_rate), + 'net_rate': float(net_rate), + 'tariffs_data': tariffs_data + }) + + return rates + + # Fallback: use average_nightly_tariff and apply to all dates + if fallback_rate and fallback_rate > 0: + net_rate = (fallback_rate / (1 + self.vat_rate)).quantize(Decimal('0.01')) + current_date = from_date + while current_date <= to_date: + # Build tariffs_data (no day-specific rates in fallback) + tariffs_data = self._build_tariffs_summary(all_tariffs_available, current_date) + rates.append({ + 'date': current_date, + 'gross_rate': float(fallback_rate), + 'net_rate': float(net_rate), + 'tariffs_data': tariffs_data + }) + current_date += timedelta(days=1) + return rates + + logger.warning(f"No rate found in response for {from_date} to {to_date}") + return rates + + def _build_tariffs_summary(self, tariffs_available: list, for_date: date = None) -> dict: + """ + Build a summary of all available tariff options for rate reporting. + + Args: + tariffs_available: List of tariff dicts from API response + for_date: Optional specific date to extract day-specific rates + + Returns: + Dict with tariff summaries - tariff_count and list of tariff details + """ + if not tariffs_available: + return {} + + summary = { + 'tariff_count': len(tariffs_available), + 'tariffs': [] + } + + date_key = for_date.isoformat() if for_date else None + + for idx, tariff in enumerate(tariffs_available): + # Get day-specific rate from tariffs_quoted if available + day_rate = None + if date_key: + tariffs_quoted = tariff.get('tariffs_quoted', {}) + if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted: + day_quote = tariffs_quoted[date_key] + if isinstance(day_quote, dict): + day_rate = float(day_quote.get('amount', 0) or 0) + else: + day_rate = float(day_quote or 0) + + # API uses tariff_label for the name + message = tariff.get('tariff_message', '') + + # Extract minimum stay from message or dedicated field + min_stay = tariff.get('minimum_nights', None) + if min_stay is None and message: + # Try to parse from message like "Minimum 2 nights" or "2 Night Minimum" + import re + match = re.search(r'(\d+)\s*[Nn]ight\s*[Mm]inimum', message) + if not match: + match = re.search(r'[Mm]inimum\s+(\d+)\s*(?:night|period)', message) + if match: + min_stay = int(match.group(1)) + + # Extract advance booking requirement from message + min_advance_days = None + if message: + import re + advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE) + if advance_match: + min_advance_days = int(advance_match.group(1)) + + tariff_info = { + 'name': tariff.get('tariff_label', 'Unknown'), + 'description': tariff.get('tariff_short_description', ''), + 'rate': day_rate, # Day-specific rate (None if not available) + 'average_nightly': float(tariff.get('average_nightly_tariff', 0) or 0), + 'success': str(tariff.get('tariff_success', False)).lower() in ('true', '1'), + 'message': message, + 'sort_order': idx, # Preserve Newbook ordering + 'min_stay': min_stay, # Minimum nights required (if any) + 'min_advance_days': min_advance_days, # Advance booking requirement (if any) + } + + summary['tariffs'].append(tariff_info) + + return summary + + def _parse_all_categories_tariffs(self, data: dict, for_date: date) -> Dict[str, List[Dict]]: + """ + Parse tariffs from API response for ALL categories. + + When category_id is omitted, data.data contains category IDs as keys, + each with their own tariffs_available. + + Args: + data: Full API response + for_date: The date we queried + + Returns: + Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]} + """ + results: Dict[str, List[Dict]] = {} + + if not isinstance(data.get("data"), dict): + return results + + for key, cat_data in data["data"].items(): + # Category IDs are numeric strings like "1", "8", etc. + if not (key.isdigit() or str(key).isnumeric()): + continue + + if not isinstance(cat_data, dict): + continue + + category_id = str(key) + tariffs_available = cat_data.get("tariffs_available", []) + + if not tariffs_available: + continue + + # Get the first (best) tariff for gross/net calculation + first_tariff = tariffs_available[0] + tariffs_quoted = first_tariff.get("tariffs_quoted", {}) + inventory_items = first_tariff.get("inventory_items", []) + + # Get rate for this date + date_key = for_date.isoformat() + gross_rate = Decimal('0') + net_rate = Decimal('0') + + if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted: + day_tariff = tariffs_quoted[date_key] + gross_rate = Decimal(str(day_tariff.get('amount', 0) or 0)) + + # Calculate included inventory per night + included_inventory = Decimal('0') + for item in inventory_items: + already_included = item.get('amount_already_included_in_tariff_total', '') + if str(already_included).lower() == 'true': + included_inventory += Decimal(str(item.get('amount', 0) or 0)) + + gross_after_inventory = gross_rate - included_inventory + net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01')) + else: + # Fallback to average + gross_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0)) + net_rate = (gross_rate / (1 + self.vat_rate)).quantize(Decimal('0.01')) + + # Build tariffs summary for all options + tariffs_data = self._build_tariffs_summary(tariffs_available, for_date) + + results[category_id] = [{ + 'date': for_date, + 'gross_rate': float(gross_rate), + 'net_rate': float(net_rate), + 'tariffs_data': tariffs_data + }] + + return results + diff --git a/backend/services/scraper_backends/__init__.py b/backend/services/scraper_backends/__init__.py new file mode 100644 index 0000000..d54c86a --- /dev/null +++ b/backend/services/scraper_backends/__init__.py @@ -0,0 +1,20 @@ +""" +Scraper backends for booking.com rate scraping. + +Provides pluggable backends to allow switching between: +- playwright_local: Direct Playwright (default) +- playwright_proxy: Playwright with rotating proxies (future) +- apify_backend: Apify scraping service (future) +""" + +from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus +from .playwright_local import PlaywrightLocalBackend + +__all__ = [ + 'ScraperBackend', + 'ScraperResult', + 'HotelData', + 'RateData', + 'AvailabilityStatus', + 'PlaywrightLocalBackend', +] diff --git a/backend/services/scraper_backends/base.py b/backend/services/scraper_backends/base.py new file mode 100644 index 0000000..3d71b0d --- /dev/null +++ b/backend/services/scraper_backends/base.py @@ -0,0 +1,152 @@ +""" +Abstract base class for booking.com scraper backends. + +Defines the interface that all scraper backends must implement, +allowing easy switching between local Playwright, proxied Playwright, +or external services like Apify. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import date +from decimal import Decimal +from typing import List, Optional, Dict, Any +from enum import Enum + + +class AvailabilityStatus(str, Enum): + """Availability status for a hotel rate.""" + AVAILABLE = 'available' # Rate found, bookable + SOLD_OUT = 'sold_out' # Hotel shows no availability + NO_DATA = 'no_data' # Couldn't determine (scraper issue) + + +@dataclass +class RateData: + """Rate data for a single hotel on a single date.""" + hotel_id: Optional[str] = None # Our internal hotel_id (filled after DB lookup) + booking_com_id: str = '' # Hotel ID from booking.com + rate_date: date = None + availability_status: AvailabilityStatus = AvailabilityStatus.NO_DATA + rate_gross: Optional[Decimal] = None + currency: str = 'GBP' + room_type: Optional[str] = None + breakfast_included: Optional[bool] = None + free_cancellation: Optional[bool] = None + no_prepayment: Optional[bool] = None + rooms_left: Optional[int] = None # "Only X rooms left" + available_qty: Optional[int] = None # Future: from hotel page dropdown + + +@dataclass +class HotelData: + """Hotel data discovered from search results.""" + booking_com_id: str + name: str + booking_com_url: Optional[str] = None + star_rating: Optional[Decimal] = None + review_score: Optional[Decimal] = None + review_count: Optional[int] = None + + +@dataclass +class ScraperResult: + """Result from a scraping operation.""" + success: bool + blocked: bool = False # True if anti-scrape blocking detected + block_reason: Optional[str] = None # CAPTCHA, rate limit, etc. + hotels: List[HotelData] = field(default_factory=list) + rates: List[RateData] = field(default_factory=list) + error_message: Optional[str] = None + page_content_sample: Optional[str] = None # For debugging + + +class ScraperBackend(ABC): + """ + Abstract base class for scraper backends. + + All backends must implement these methods to provide a consistent + interface for the main booking_scraper.py service. + """ + + # Common block detection signals + BLOCK_SIGNALS = [ + 'captcha', + 'unusual traffic', + 'access denied', + 'please verify', + 'too many requests', + 'are you a robot', + 'verify you are human', + 'security check', + ] + + @abstractmethod + async def scrape_location_search( + self, + location: str, + check_in: date, + check_out: date, + adults: int = 2, + pages: int = 2 + ) -> ScraperResult: + """ + Scrape booking.com location search results. + + Args: + location: Location name (e.g., "Bowness-on-Windermere") + check_in: Check-in date + check_out: Check-out date (typically check_in + 1 for single night) + adults: Number of adults for search + pages: Number of search result pages to scrape + + Returns: + ScraperResult with hotels and rates found + """ + pass + + @abstractmethod + async def scrape_hotel_page( + self, + hotel_url: str, + check_in: date, + check_out: date, + adults: int = 2 + ) -> ScraperResult: + """ + Scrape an individual hotel page for detailed rates. + + Future expansion - not used in initial implementation. + Will provide available_qty from room dropdowns. + + Args: + hotel_url: Full booking.com URL for the hotel + check_in: Check-in date + check_out: Check-out date + adults: Number of adults + + Returns: + ScraperResult with detailed rate information + """ + pass + + @abstractmethod + async def close(self): + """Clean up any resources (browser instances, etc.).""" + pass + + def detect_blocking(self, page_content: str) -> tuple[bool, Optional[str]]: + """ + Check if page content shows anti-scrape response. + + Args: + page_content: HTML content of the page + + Returns: + Tuple of (is_blocked, reason) + """ + content_lower = page_content.lower() + for signal in self.BLOCK_SIGNALS: + if signal in content_lower: + return True, signal + return False, None diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py new file mode 100644 index 0000000..dfe6540 --- /dev/null +++ b/backend/services/scraper_backends/playwright_local.py @@ -0,0 +1,401 @@ +""" +Local Playwright backend for booking.com scraping. + +Uses Playwright with Chromium to scrape search results. +No proxy - direct connection. Suitable for low-volume scraping. +""" + +import asyncio +import logging +import random +import re +from datetime import date +from decimal import Decimal, InvalidOperation +from typing import List, Optional +from urllib.parse import urlencode + +from playwright.async_api import async_playwright, Browser, BrowserContext, Page + +from .base import ( + ScraperBackend, + ScraperResult, + HotelData, + RateData, + AvailabilityStatus +) + +logger = logging.getLogger(__name__) + + +class PlaywrightLocalBackend(ScraperBackend): + """ + Local Playwright backend using Chromium. + + Features: + - Rotates user agents + - Random delays between requests + - Mimics human scroll behavior + - Uses data-testid selectors for stability + """ + + USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0", + ] + + def __init__(self, proxy_config: dict = None): + """ + Initialize the backend. + + Args: + proxy_config: Optional proxy configuration (for future use) + """ + self.proxy_config = proxy_config + self._playwright = None + self._browser: Optional[Browser] = None + + async def _ensure_browser(self) -> Browser: + """Ensure browser is running, start if needed.""" + if self._browser is None or not self._browser.is_connected(): + self._playwright = await async_playwright().start() + self._browser = await self._playwright.chromium.launch( + headless=True, + args=[ + '--disable-blink-features=AutomationControlled', + '--no-sandbox', + '--disable-dev-shm-usage', + ] + ) + return self._browser + + async def _create_context(self) -> BrowserContext: + """Create a new browser context with random user agent.""" + browser = await self._ensure_browser() + context = await browser.new_context( + user_agent=random.choice(self.USER_AGENTS), + viewport={'width': 1920, 'height': 1080}, + locale='en-GB', + timezone_id='Europe/London', + ) + return context + + def _build_search_url( + self, + location: str, + check_in: date, + check_out: date, + adults: int, + offset: int = 0 + ) -> str: + """Build booking.com search URL with parameters.""" + params = { + 'ss': location, + 'checkin': check_in.isoformat(), + 'checkout': check_out.isoformat(), + 'group_adults': adults, + 'no_rooms': 1, + 'group_children': 0, + } + if offset > 0: + params['offset'] = offset + + return f"https://www.booking.com/searchresults.en-gb.html?{urlencode(params)}" + + def _parse_price(self, price_text: str) -> Optional[Decimal]: + """Parse price from text like '£150' or 'GBP 150'.""" + if not price_text: + return None + # Remove currency symbols and extract number + cleaned = re.sub(r'[£$€,\s]', '', price_text) + # Find first number (including decimals) + match = re.search(r'[\d,]+(?:\.\d{2})?', cleaned) + if match: + try: + return Decimal(match.group().replace(',', '')) + except InvalidOperation: + return None + return None + + def _extract_hotel_id(self, url: str) -> Optional[str]: + """Extract hotel ID from booking.com URL.""" + if not url: + return None + # URL format: /hotel/gb/hotel-name.en-gb.html or ?dest_id=123 + # Try to extract from URL path + match = re.search(r'/hotel/[a-z]{2}/([^/]+)\.', url) + if match: + return match.group(1) + # Try dest_id parameter + match = re.search(r'dest_id=(-?\d+)', url) + if match: + return match.group(1) + return None + + async def _human_like_scroll(self, page: Page): + """Simulate human-like scrolling behavior.""" + # Scroll down in increments + for _ in range(3): + await page.mouse.wheel(0, random.randint(300, 600)) + await asyncio.sleep(random.uniform(0.3, 0.8)) + + async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]: + """Extract hotel and rate data from search results page.""" + hotels = [] + rates = [] + + # Wait for property cards - booking.com uses data-testid + try: + await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) + except Exception as e: + logger.warning(f"No property cards found: {e}") + return hotels, rates + + # Get all property cards + cards = await page.query_selector_all('[data-testid="property-card"]') + logger.info(f"Found {len(cards)} property cards") + + for card in cards: + try: + hotel = HotelData(booking_com_id='', name='') + rate = RateData(rate_date=rate_date) + + # Hotel name + name_el = await card.query_selector('[data-testid="title"]') + if name_el: + hotel.name = (await name_el.inner_text()).strip() + + if not hotel.name: + continue # Skip if no name found + + # Hotel URL and ID + link_el = await card.query_selector('[data-testid="title-link"]') + if link_el: + hotel.booking_com_url = await link_el.get_attribute('href') + hotel.booking_com_id = self._extract_hotel_id(hotel.booking_com_url) or '' + + rate.booking_com_id = hotel.booking_com_id + + # Star rating - look for star icons or rating text + stars_el = await card.query_selector('[data-testid="rating-stars"]') + if stars_el: + stars_text = await stars_el.get_attribute('aria-label') or '' + match = re.search(r'(\d+)', stars_text) + if match: + hotel.star_rating = Decimal(match.group(1)) + + # Review score + score_el = await card.query_selector('[data-testid="review-score"]') + if score_el: + score_text = await score_el.inner_text() + match = re.search(r'([\d.]+)', score_text) + if match: + try: + hotel.review_score = Decimal(match.group(1)) + except InvalidOperation: + pass + + # Check for no availability message FIRST + no_avail_el = await card.query_selector('[data-testid="availability-message"]') + if no_avail_el: + avail_text = (await no_avail_el.inner_text()).lower() + if 'no availability' in avail_text or 'sold out' in avail_text: + rate.availability_status = AvailabilityStatus.SOLD_OUT + hotels.append(hotel) + rates.append(rate) + continue + + # Price + price_el = await card.query_selector('[data-testid="price-and-discounted-price"]') + if not price_el: + # Try alternative selector + price_el = await card.query_selector('[data-testid="price"]') + + if price_el: + price_text = await price_el.inner_text() + rate.rate_gross = self._parse_price(price_text) + if rate.rate_gross: + rate.availability_status = AvailabilityStatus.AVAILABLE + + # Room type + room_el = await card.query_selector('[data-testid="recommended-units"]') + if room_el: + rate.room_type = (await room_el.inner_text()).strip() + + # Rate option badges - try multiple selectors + # Breakfast included + breakfast_el = await card.query_selector('[data-testid="breakfast-included"]') + if not breakfast_el: + # Check text content for breakfast mentions + card_text = (await card.inner_text()).lower() + rate.breakfast_included = 'breakfast included' in card_text + else: + rate.breakfast_included = True + + # Free cancellation + cancel_el = await card.query_selector('[data-testid="cancellation-policy"]') + if cancel_el: + cancel_text = (await cancel_el.inner_text()).lower() + rate.free_cancellation = 'free cancellation' in cancel_text + else: + card_text = (await card.inner_text()).lower() + rate.free_cancellation = 'free cancellation' in card_text + + # No prepayment + prepay_el = await card.query_selector('[data-testid="no-prepayment"]') + if prepay_el: + rate.no_prepayment = True + else: + card_text = (await card.inner_text()).lower() + rate.no_prepayment = 'no prepayment' in card_text + + # Rooms left / scarcity indicator + scarcity_el = await card.query_selector('[data-testid="availability-rate"]') + if scarcity_el: + scarcity_text = await scarcity_el.inner_text() + match = re.search(r'(\d+)\s*room', scarcity_text.lower()) + if match: + rate.rooms_left = int(match.group(1)) + + hotels.append(hotel) + rates.append(rate) + + except Exception as e: + logger.warning(f"Error extracting hotel data: {e}") + continue + + return hotels, rates + + async def scrape_location_search( + self, + location: str, + check_in: date, + check_out: date, + adults: int = 2, + pages: int = 2 + ) -> ScraperResult: + """ + Scrape booking.com location search results. + + Args: + location: Location name + check_in: Check-in date + check_out: Check-out date (check_in + 1 for single night rate) + adults: Number of adults + pages: Number of result pages to scrape + + Returns: + ScraperResult with hotels and rates found + """ + all_hotels = [] + all_rates = [] + seen_hotel_ids = set() + + context = None + page = None + + try: + context = await self._create_context() + page = await context.new_page() + + for page_num in range(pages): + # Random delay between pages (3-7 seconds) + if page_num > 0: + delay = random.uniform(3, 7) + logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}") + await asyncio.sleep(delay) + + # Build URL with offset for pagination (25 results per page) + url = self._build_search_url( + location, check_in, check_out, adults, + offset=page_num * 25 + ) + + logger.info(f"Scraping page {page_num + 1}: {url}") + + try: + await page.goto(url, wait_until='networkidle', timeout=30000) + except Exception as e: + logger.warning(f"Page load timeout, continuing: {e}") + + # Check for blocking + content = await page.content() + is_blocked, reason = self.detect_blocking(content) + if is_blocked: + logger.warning(f"Blocking detected: {reason}") + return ScraperResult( + success=False, + blocked=True, + block_reason=reason, + hotels=all_hotels, + rates=all_rates, + page_content_sample=content[:1000] + ) + + # Human-like scrolling + await self._human_like_scroll(page) + + # Extract data + hotels, rates = await self._extract_search_results(page, check_in) + + # Deduplicate by booking_com_id + for hotel, rate in zip(hotels, rates): + if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids: + seen_hotel_ids.add(hotel.booking_com_id) + all_hotels.append(hotel) + all_rates.append(rate) + + logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique") + + return ScraperResult( + success=True, + blocked=False, + hotels=all_hotels, + rates=all_rates + ) + + except Exception as e: + logger.error(f"Scrape error: {e}") + return ScraperResult( + success=False, + blocked=False, + error_message=str(e), + hotels=all_hotels, + rates=all_rates + ) + finally: + if page: + await page.close() + if context: + await context.close() + + async def scrape_hotel_page( + self, + hotel_url: str, + check_in: date, + check_out: date, + adults: int = 2 + ) -> ScraperResult: + """ + Scrape individual hotel page for detailed rates. + + Future expansion - placeholder for now. + Will extract available_qty from room dropdowns. + """ + # Not implemented in Phase 2a + logger.warning("scrape_hotel_page not yet implemented") + return ScraperResult( + success=False, + error_message="Hotel page scraping not yet implemented" + ) + + async def close(self): + """Clean up browser resources.""" + if self._browser: + await self._browser.close() + self._browser = None + if self._playwright: + await self._playwright.stop() + self._playwright = None diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0fbd5fd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +services: + backend: + build: ./backend + security_opt: + - apparmor=unconfined + environment: + - DATABASE_URL=${DATABASE_URL} + - CENTRAL_AUTH_SECRET=${CENTRAL_AUTH_SECRET} + - APP_SLUG=rates + - SETTINGS_URL=${SETTINGS_URL:-} + - SETTINGS_SECRET=${SETTINGS_SECRET:-} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 60s + restart: unless-stopped + + frontend: + build: + context: ./frontend + args: + VITE_HOTEL_NAME: ${VITE_HOTEL_NAME:-Hotel} + security_opt: + - apparmor=unconfined + ports: + - "${FRONTEND_PORT:-3080}:80" + depends_on: + backend: + condition: service_healthy + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..16ec634 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,15 @@ +FROM node:22-alpine AS builder + +ARG VITE_HOTEL_NAME=Hotel +ENV VITE_HOTEL_NAME=$VITE_HOTEL_NAME + +WORKDIR /app +COPY package.json ./ +RUN npm install +COPY . . +RUN npm run build + +FROM nginx:alpine +COPY --from=builder /app/dist /usr/share/nginx/html/rates +COPY nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..88bea8d --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + Rate Monitor + + +
+ + + diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 0000000..6c792aa --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,35 @@ +server { + listen 80; + server_name _; + + # 1. Central auth proxy + location /rates/api/auth/ { + proxy_pass http://10.10.10.101:3001/api/auth/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # 2. App backend (Python FastAPI on port 8000) + location /rates/api/ { + proxy_pass http://backend:8000/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Cookie $http_cookie; + proxy_read_timeout 300s; + proxy_connect_timeout 30s; + client_max_body_size 10M; + } + + # 3. Health + location /rates/health { + proxy_pass http://backend:8000/health; + } + + # 4. SPA fallback + location /rates/ { + root /usr/share/nginx/html; + try_files $uri $uri/ /rates/index.html; + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..d7e9775 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,30 @@ +{ + "name": "rates-frontend", + "private": true, + "version": "1.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@tanstack/react-query": "^5.32.0", + "axios": "^1.6.8", + "lucide-react": "^0.395.0", + "plotly.js": "^2.29.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-plotly.js": "^2.6.0", + "react-router-dom": "^6.22.0" + }, + "devDependencies": { + "@types/plotly.js": "^2.12.29", + "@types/react": "^18.3.1", + "@types/react-dom": "^18.3.0", + "@types/react-plotly.js": "^2.6.4", + "@vitejs/plugin-react": "^4.2.1", + "typescript": "^5.4.5", + "vite": "^5.2.11" + } +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..e683936 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,28 @@ +import { Routes, Route, Navigate } from 'react-router-dom' +import AuthGate from './components/AuthGate' +import Layout from './components/Layout' +import Bookability from './pages/Bookability' +import MarketView from './pages/MarketView' +import DirectRates from './pages/DirectRates' +import RateAnalysis from './pages/RateAnalysis' +import Settings from './pages/Settings' + +export default function App() { + return ( + + + + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + + ) +} diff --git a/frontend/src/api.ts b/frontend/src/api.ts new file mode 100644 index 0000000..6419301 --- /dev/null +++ b/frontend/src/api.ts @@ -0,0 +1,20 @@ +import axios from 'axios' + +const BASE = '/rates/api' + +const api = axios.create({ + baseURL: BASE, + withCredentials: true, +}) + +api.interceptors.response.use( + (response) => response, + (error) => { + if (error.response?.status === 401) { + window.location.href = '/auth/login?redirect=' + encodeURIComponent(window.location.pathname) + } + return Promise.reject(error) + } +) + +export default api diff --git a/frontend/src/components/AuthGate.tsx b/frontend/src/components/AuthGate.tsx new file mode 100644 index 0000000..7fa8b9c --- /dev/null +++ b/frontend/src/components/AuthGate.tsx @@ -0,0 +1,47 @@ +import { createContext, useContext, useEffect, useState } from 'react' +import type { ReactNode } from 'react' +import type { User } from '../types' + +interface AuthCtx { user: User } +const Ctx = createContext(null) + +export function useAuth() { + const ctx = useContext(Ctx) + if (!ctx) throw new Error('useAuth outside AuthGate') + return ctx +} + +export default function AuthGate({ children }: { children: ReactNode }) { + const [user, setUser] = useState(null) + const [checking, setChecking] = useState(true) + + useEffect(() => { + fetch('/rates/api/auth/verify?app=rates', { credentials: 'include' }) + .then(r => { + if (!r.ok) throw new Error('unauth') + return r.json() + }) + .then(data => setUser({ + email: data.email || data.sub || '', + name: data.name || data.display_name || '', + is_admin: data.is_admin ?? false, + caps: data.caps ?? [], + })) + .catch(() => { + window.location.href = '/auth/login?redirect=' + encodeURIComponent(window.location.pathname) + }) + .finally(() => setChecking(false)) + }, []) + + if (checking) { + return ( +
+
+
+ ) + } + + if (!user) return null + + return {children} +} diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx new file mode 100644 index 0000000..03c3c6b --- /dev/null +++ b/frontend/src/components/Layout.tsx @@ -0,0 +1,70 @@ +import { NavLink } from 'react-router-dom' +import { TrendingUp, Calendar, Globe, BarChart2, Settings, Building2 } from 'lucide-react' +import { useQuery } from '@tanstack/react-query' +import { useAuth } from './AuthGate' +import { can } from '../types' +import api from '../api' +import type { ReactNode } from 'react' + +const ICON = { size: 16, strokeWidth: 1.75 } + +const NAV = [ + { to: '/bookability', label: 'Bookability', icon: Calendar, cap: 'view_own_rates' }, + { to: '/market', label: 'Market View', icon: Globe, cap: 'view_competitors' }, + { to: '/direct', label: 'Direct Rates', icon: Building2, cap: 'view_direct_rates' }, + { to: '/analysis', label: 'Rate Analysis', icon: TrendingUp, cap: 'rate_analysis' }, + { to: '/settings', label: 'Settings', icon: Settings, cap: 'manage_scraper' }, +] + +export default function Layout({ children }: { children: ReactNode }) { + const { user } = useAuth() + const items = NAV.filter(n => can(user, n.cap)) + + const { data: alertCount } = useQuery({ + queryKey: ['parity-alert-count'], + queryFn: () => api.get('/competitors/parity/alerts?status=active').then(r => r.data.length), + refetchInterval: 60_000, + enabled: can(user, 'view_competitors'), + }) + + return ( +
+ + +
+ + Rate Monitor + +
+ +
{children}
+
+ ) +} diff --git a/frontend/src/index.css b/frontend/src/index.css new file mode 100644 index 0000000..1c0146a --- /dev/null +++ b/frontend/src/index.css @@ -0,0 +1,363 @@ +:root { + --navy: #1a1a2e; + --navy-dark: #0f0f20; + --gold: #c9a84c; + --gold-light: #e8c96d; + --surface: rgba(255,255,255,0.07); + --surface-2: rgba(255,255,255,0.08); + --text: rgba(255,255,255,0.88); + --text-muted: rgba(255,255,255,0.48); + --body-bg: #f4f5f7; + --card-bg: #ffffff; + --card-border: #e4e8ee; + --text-dark: #1e293b; + --text-mid: #64748b; + --shadow-sm: 0 1px 3px rgba(0,0,0,0.07), 0 1px 2px rgba(0,0,0,0.04); + --shadow-md: 0 4px 12px rgba(0,0,0,0.08); + --danger: #dc2626; + --success: #16a34a; + --warning: #d97706; + --radius: 10px; + --font: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; +} + +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +body { + background: var(--body-bg); + color: var(--text-dark); + font-family: var(--font); + font-size: 14px; + line-height: 1.5; +} + +/* App shell layout */ +.app-shell { + display: grid; + grid-template-columns: 220px 1fr; + grid-template-rows: auto 1fr; + min-height: 100vh; +} + +.sidebar { + grid-column: 1; + grid-row: 1 / -1; + background: var(--navy); + display: flex; + flex-direction: column; + padding: 0; + position: sticky; + top: 0; + height: 100vh; + overflow-y: auto; +} + +.sidebar-logo { + display: flex; + align-items: center; + gap: 10px; + padding: 20px 16px 16px; + font-size: 15px; + font-weight: 600; + color: var(--text); + border-bottom: 1px solid var(--surface); +} + +.sidebar-nav { + flex: 1; + padding: 12px 8px; + display: flex; + flex-direction: column; + gap: 2px; +} + +.sidebar-nav a { + display: flex; + align-items: center; + gap: 10px; + padding: 9px 10px; + border-radius: 8px; + color: var(--text-muted); + text-decoration: none; + font-size: 13.5px; + transition: background 0.15s, color 0.15s; +} + +.sidebar-nav a:hover { background: var(--surface); color: var(--text); } +.sidebar-nav a.active { background: rgba(201,168,76,0.15); color: var(--gold); } + +.sidebar-user { + padding: 12px 16px; + font-size: 12px; + color: var(--text-muted); + border-top: 1px solid var(--surface); +} + +/* Top bar — mobile/collapsed fallback */ +.top-bar { + display: none; +} + +.page-content { + grid-column: 2; + grid-row: 1 / -1; + min-width: 0; + padding: 24px; + background: var(--body-bg); +} + +/* Cards */ +.card { + background: var(--card-bg); + border: 1px solid var(--card-border); + border-radius: var(--radius); + box-shadow: var(--shadow-sm); +} + +.card-header { + padding: 16px 20px; + border-bottom: 1px solid var(--card-border); + font-size: 14px; + font-weight: 600; + color: var(--text-dark); + display: flex; + align-items: center; + justify-content: space-between; +} + +.card-body { padding: 20px; } + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 7px 14px; + border-radius: 7px; + font-size: 13px; + font-weight: 500; + cursor: pointer; + border: none; + transition: background 0.15s, opacity 0.15s; +} + +.btn:disabled { opacity: 0.5; cursor: not-allowed; } + +.btn-primary { + background: var(--gold); + color: var(--navy); +} +.btn-primary:hover:not(:disabled) { background: var(--gold-light); } + +.btn-secondary { + background: var(--navy); + color: var(--text); +} +.btn-secondary:hover:not(:disabled) { background: var(--navy-dark); } + +.btn-outline { + background: transparent; + color: var(--text-dark); + border: 1px solid var(--card-border); +} +.btn-outline:hover:not(:disabled) { background: var(--body-bg); } + +.btn-danger { + background: var(--danger); + color: white; +} +.btn-danger:hover:not(:disabled) { opacity: 0.85; } + +.btn-sm { padding: 4px 10px; font-size: 12px; } + +/* Form elements */ +input, select, textarea { + background: var(--card-bg); + border: 1px solid var(--card-border); + border-radius: 7px; + padding: 7px 10px; + font-size: 13px; + color: var(--text-dark); + width: 100%; + outline: none; + transition: border-color 0.15s; +} +input:focus, select:focus, textarea:focus { border-color: var(--gold); } + +/* Badges */ +.badge { + display: inline-flex; + align-items: center; + padding: 2px 8px; + border-radius: 20px; + font-size: 11px; + font-weight: 500; +} +.badge-success { background: #dcfce7; color: #16a34a; } +.badge-warning { background: #fef3c7; color: #d97706; } +.badge-danger { background: #fee2e2; color: #dc2626; } +.badge-info { background: #dbeafe; color: #2563eb; } +.badge-neutral { background: #f1f5f9; color: #64748b; } + +/* Tables */ +.table-wrap { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; font-size: 13px; } +th { + text-align: left; + padding: 8px 12px; + background: #f8fafc; + border-bottom: 1px solid var(--card-border); + font-weight: 600; + color: var(--text-mid); + white-space: nowrap; +} +td { + padding: 8px 12px; + border-bottom: 1px solid #f1f5f9; + color: var(--text-dark); +} +tr:last-child td { border-bottom: none; } +tr:hover td { background: #f8fafc; } + +/* Page header */ +.page-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 24px; + gap: 16px; +} +.page-title { + font-size: 20px; + font-weight: 700; + color: var(--text-dark); +} +.page-subtitle { font-size: 13px; color: var(--text-mid); margin-top: 2px; } + +/* Sub-nav tabs */ +.sub-nav { + display: flex; + gap: 4px; + border-bottom: 1px solid var(--card-border); + margin-bottom: 24px; + overflow-x: auto; +} +.sub-nav-item { + padding: 8px 14px; + font-size: 13px; + font-weight: 500; + color: var(--text-mid); + cursor: pointer; + border-bottom: 2px solid transparent; + white-space: nowrap; + background: none; + border-left: none; + border-right: none; + border-top: none; + transition: color 0.15s, border-color 0.15s; +} +.sub-nav-item:hover { color: var(--text-dark); } +.sub-nav-item.active { color: var(--gold); border-bottom-color: var(--gold); } + +/* Status dot */ +.status-dot { + width: 8px; height: 8px; + border-radius: 50%; + display: inline-block; +} +.status-dot.green { background: var(--success); } +.status-dot.yellow { background: var(--warning); } +.status-dot.red { background: var(--danger); } +.status-dot.grey { background: #94a3b8; } + +/* Spinner */ +.spinner { + width: 20px; height: 20px; + border: 2px solid var(--card-border); + border-top-color: var(--gold); + border-radius: 50%; + animation: spin 0.7s linear infinite; + display: inline-block; +} +@keyframes spin { to { transform: rotate(360deg); } } + +/* Loading / empty states */ +.loading-state { + display: flex; + align-items: center; + justify-content: center; + gap: 10px; + padding: 60px 20px; + color: var(--text-mid); + font-size: 13px; +} + +.empty-state { + text-align: center; + padding: 60px 20px; + color: var(--text-mid); + font-size: 13px; +} + +/* Grid helpers */ +.grid-2 { display: grid; grid-template-columns: repeat(2, 1fr); gap: 16px; } +.grid-3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; } +.grid-4 { display: grid; grid-template-columns: repeat(4, 1fr); gap: 16px; } + +/* Stat card */ +.stat-card { + background: var(--card-bg); + border: 1px solid var(--card-border); + border-radius: var(--radius); + padding: 16px 20px; + box-shadow: var(--shadow-sm); +} +.stat-label { font-size: 12px; color: var(--text-mid); margin-bottom: 4px; } +.stat-value { font-size: 24px; font-weight: 700; color: var(--text-dark); } +.stat-delta { font-size: 12px; margin-top: 4px; } +.stat-delta.positive { color: var(--success); } +.stat-delta.negative { color: var(--danger); } +.stat-delta.neutral { color: var(--text-mid); } + +/* Responsive — at narrow widths hide sidebar, show top-bar */ +@media (max-width: 900px) { + .app-shell { + grid-template-columns: 1fr; + grid-template-rows: auto 1fr; + } + .sidebar { display: none; } + .top-bar { + display: flex; + align-items: center; + gap: 12px; + background: var(--navy); + padding: 0 16px; + height: 52px; + grid-column: 1; + grid-row: 1; + overflow-x: auto; + } + .top-bar-title { + font-size: 14px; + font-weight: 600; + color: var(--text); + white-space: nowrap; + margin-right: 8px; + } + .top-bar-nav { display: flex; gap: 4px; } + .top-bar-nav a { + color: var(--text-muted); + text-decoration: none; + font-size: 12.5px; + padding: 6px 10px; + border-radius: 6px; + white-space: nowrap; + } + .top-bar-nav a:hover { color: var(--text); background: var(--surface); } + .top-bar-nav a.active { color: var(--gold); } + .page-content { + grid-column: 1; + grid-row: 2; + padding: 16px; + } +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 0000000..6343c88 --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,20 @@ +import React from 'react' +import ReactDOM from 'react-dom/client' +import { BrowserRouter } from 'react-router-dom' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import App from './App' +import './index.css' + +const qc = new QueryClient({ + defaultOptions: { queries: { retry: 1, staleTime: 30_000 } }, +}) + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + + + + + , +) diff --git a/frontend/src/pages/Bookability.tsx b/frontend/src/pages/Bookability.tsx new file mode 100644 index 0000000..7db965a --- /dev/null +++ b/frontend/src/pages/Bookability.tsx @@ -0,0 +1,1094 @@ +import React, { useState, useMemo } from 'react' +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { Link } from 'react-router-dom' +import api from '../api' + +// Format Date as YYYY-MM-DD using local time (avoids UTC/DST shift from toISOString) +const fmtDate = (d: Date) => `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` + +// Types +interface CategoryInfo { + category_id: string + category_name: string + room_count: number +} + +interface TariffInfo { + name: string + description?: string + rate: number | null + average_nightly?: number + available: boolean + message: string + sort_order?: number + min_stay?: number | null + available_for_min_stay?: boolean | null // True if available when queried with min_stay nights +} + +interface OccupancyInfo { + occupied: number + available: number + maintenance: number +} + +interface DateRateInfo { + rate_gross: number | null + rate_net: number | null + tariffs: TariffInfo[] + tariff_count: number + occupancy?: OccupancyInfo + valid_from?: string | null +} + +interface RateMatrixData { + categories: CategoryInfo[] + dates: string[] + matrix: Record> + date_last_updated?: Record +} + +// Helper functions +const formatDateShort = (dateStr: string): string => { + const date = new Date(dateStr + 'T00:00:00') + return date.toLocaleDateString('en-GB', { day: 'numeric' }) +} + +const formatDayOfWeek = (dateStr: string): string => { + const date = new Date(dateStr + 'T00:00:00') + return date.toLocaleDateString('en-GB', { weekday: 'short' }) +} + +const formatLastUpdated = (isoStr: string | null | undefined): string => { + if (!isoStr) return '' + const d = new Date(isoStr) + const now = new Date() + const isToday = d.toDateString() === now.toDateString() + return isToday + ? d.toLocaleTimeString('en-GB', { hour: '2-digit', minute: '2-digit' }) + : d.toLocaleDateString('en-GB', { day: '2-digit', month: 'short' }) + ' ' + d.toLocaleTimeString('en-GB', { hour: '2-digit', minute: '2-digit' }) +} + +const isWeekend = (dateStr: string): boolean => { + const date = new Date(dateStr + 'T00:00:00') + const day = date.getDay() + return day === 0 || day === 6 +} + +const formatCurrency = (value: number | null): string => { + if (value === null || value === undefined) return '-' + return new Intl.NumberFormat('en-GB', { + style: 'currency', + currency: 'GBP', + minimumFractionDigits: 0, + maximumFractionDigits: 0, + }).format(value) +} + +// Inline style helpers (replacing theme utilities) +const mergeStyles = (...styles: React.CSSProperties[]): React.CSSProperties => + Object.assign({}, ...styles) + +const buttonStyle = (variant: 'primary' | 'secondary' | 'outline', size?: 'small'): React.CSSProperties => { + const base: React.CSSProperties = { + border: 'none', + borderRadius: '6px', + cursor: 'pointer', + fontWeight: 500, + padding: size === 'small' ? '4px 10px' : '8px 16px', + fontSize: size === 'small' ? '13px' : '14px', + lineHeight: 1.4, + transition: 'all 0.15s', + } + if (variant === 'primary') return { ...base, background: 'var(--gold)', color: '#fff' } + if (variant === 'secondary') return { ...base, background: 'var(--navy)', color: '#fff' } + // outline + return { ...base, background: 'transparent', color: 'var(--text-dark)', border: '1px solid var(--card-border)' } +} + +const badgeStyle = (variant: 'success' | 'error' | 'warning' | 'info'): React.CSSProperties => { + const map: Record = { + success: { background: '#dcfce7', color: '#16a34a', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + error: { background: '#fee2e2', color: '#dc2626', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + warning: { background: '#fef3c7', color: '#d97706', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + info: { background: '#dbeafe', color: '#2563eb', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + } + return map[variant] || map.info +} + +// Components +const MonthSelector: React.FC<{ + value: string + onChange: (value: string) => void +}> = ({ value, onChange }) => { + const handlePrevMonth = () => { + const [year, month] = value.split('-').map(Number) + const date = new Date(year, month - 2, 1) + onChange(`${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`) + } + + const handleNextMonth = () => { + const [year, month] = value.split('-').map(Number) + const date = new Date(year, month, 1) + onChange(`${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`) + } + + // Generate month options (current month + next 12 months) + const monthOptions = useMemo(() => { + const options: { value: string; label: string }[] = [] + const now = new Date() + for (let i = 0; i < 13; i++) { + const date = new Date(now.getFullYear(), now.getMonth() + i, 1) + const monthValue = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}` + const label = date.toLocaleDateString('en-GB', { month: 'long', year: 'numeric' }) + options.push({ value: monthValue, label }) + } + return options + }, []) + + return ( +
+ + + +
+ ) +} + +// Helper to collect unique tariff names across all dates for a category, preserving Newbook order +const getAllTariffNames = (rateData: Record, dates: string[]): string[] => { + const tariffMap = new Map() + for (const dateStr of dates) { + const data = rateData[dateStr] + if (data?.tariffs) { + for (const tariff of data.tariffs) { + if (!tariffMap.has(tariff.name)) { + tariffMap.set(tariff.name, tariff.sort_order ?? 999) + } + } + } + } + return Array.from(tariffMap.entries()) + .sort((a, b) => a[1] - b[1]) + .map(([name]) => name) +} + +// Helper to format scrape age +const formatScrapeAge = (isoStr: string | null): string => { + if (!isoStr) return '' + const diff = Date.now() - new Date(isoStr).getTime() + const mins = Math.floor(diff / 60000) + if (mins < 60) return `${mins}m ago` + const hours = Math.floor(mins / 60) + if (hours < 24) return `${hours}h ago` + const days = Math.floor(hours / 24) + return `${days}d ago` +} + +interface BookingAvailabilityData { + has_own_hotel: boolean + dates_checked: number + dates_available: number + dates_sold_out: number + dates_no_data: number + latest_scrape: string | null + dates: Record +} + +const LoadingSpinner: React.FC = () => ( +
+
+ Loading rate data... +
+) + +const ErrorMessage: React.FC<{ message: string }> = ({ message }) => ( +
+ ! + {message} +
+) + +// Main Component +const Bookability: React.FC = () => { + const queryClient = useQueryClient() + const [selectedMonth, setSelectedMonth] = useState(() => { + const today = new Date() + return `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}` + }) + const [refreshingDate, setRefreshingDate] = useState(null) + + // Calculate date range from selected month + const { fromDate, toDate } = useMemo(() => { + const [year, month] = selectedMonth.split('-').map(Number) + const start = new Date(year, month - 1, 1) + const end = new Date(year, month, 0) // Last day of month + return { + fromDate: fmtDate(start), + toDate: fmtDate(end), + } + }, [selectedMonth]) + + // Single-date refresh mutation + const dateRefreshM = useMutation({ + mutationFn: async (d: string) => { + setRefreshingDate(d) + const res = await api.post(`/bookability/refresh-date/${d}`) + return res.data + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['rate-matrix'] }) + setRefreshingDate(null) + }, + onError: () => setRefreshingDate(null), + }) + + // Fetch Booking.com availability data + const { data: bookingData } = useQuery({ + queryKey: ['booking-availability', fromDate, toDate], + queryFn: async () => { + const params = new URLSearchParams({ from_date: fromDate, to_date: toDate }) + const res = await api.get(`/competitor-rates/booking-availability?${params}`) + return res.data + }, + staleTime: 5 * 60 * 1000, + }) + + // Fetch rate matrix data + const { data, isLoading, error, refetch } = useQuery({ + queryKey: ['rate-matrix', fromDate, toDate], + queryFn: async () => { + const params = new URLSearchParams({ from_date: fromDate, to_date: toDate }) + const res = await api.get(`/bookability/rate-matrix?${params}`) + return res.data + }, + }) + + // Calculate summary stats - focus on unbookable dates (rooms available but no rates) + const summary = useMemo(() => { + if (!data) return null + + let totalDateCategories = 0 + let unbookableDateCategories = 0 + const unbookableIssues: { category: string; date: string; roomsLeft: number }[] = [] + + for (const cat of data.categories) { + const catData = data.matrix[cat.category_id] + if (!catData) continue + + for (const dateStr of data.dates) { + const dayData = catData[dateStr] + if (!dayData) continue + + // Check if rooms are available (bookable = available - maintenance - occupied) + const occ = dayData.occupancy + const bookableRooms = occ ? occ.available - occ.maintenance : 0 + const roomsLeft = occ ? bookableRooms - occ.occupied : 0 + const hasRoomsAvailable = roomsLeft > 0 + + // Only count dates where rooms are available + if (hasRoomsAvailable) { + totalDateCategories++ + + // Check if ANY tariff is available for booking + // A tariff is "bookable" if: + // - available: true (single-night available), OR + // - has min_stay > 1 AND available_for_min_stay: true (verified via multi-night query) + const hasAnyAvailableRate = dayData.tariffs?.some(t => { + if (t.available) return true + // If has min_stay requirement and verified available for that stay length + if (t.min_stay && t.min_stay > 1 && t.available_for_min_stay === true) return true + return false + }) ?? false + + if (!hasAnyAvailableRate && dayData.tariffs && dayData.tariffs.length > 0) { + // Rooms available but no rates bookable - this is a problem! + unbookableDateCategories++ + unbookableIssues.push({ + category: cat.category_name, + date: dateStr, + roomsLeft: roomsLeft, + }) + } + } + } + } + + return { + totalDateCategories, + unbookableDateCategories, + unbookablePercent: totalDateCategories > 0 + ? ((unbookableDateCategories / totalDateCategories) * 100).toFixed(1) + : '0', + issues: unbookableIssues.slice(0, 10), + hasMoreIssues: unbookableIssues.length > 10, + totalIssues: unbookableIssues.length, + } + }, [data]) + + return ( +
+ {/* Header */} +
+
+
+

Rate Availability

+

+ View tariff availability across all room categories +

+
+
+ + +
+
+ + {/* Summary Stats */} + {summary && ( +
+
+ {data?.categories.length || 0} + Room Types +
+
+ {data?.dates.length || 0} + Days +
+
+ 0 ? { color: 'var(--danger)' } : { color: 'var(--success)' } + )}> + {summary.unbookableDateCategories} + + Unbookable +
+
+ 0 ? 'warning' : 'success')}> + {summary.unbookablePercent}% blocked + +
+
+ )} +
+ + {/* Content */} +
+ {isLoading && } + {error && } + {data && data.categories.length === 0 && ( +
+ No room categories configured. Please set up room categories in Settings. +
+ )} + {data && data.categories.length > 0 && ( +
+
+ + + + + {data.dates.map(dateStr => ( + + ))} + + + + {data.categories.map(category => { + const rateData = data.matrix[category.category_id] || {} + const tariffNames = getAllTariffNames(rateData, data.dates) + return ( + + {/* Category header row */} + + + {data.dates.map(dateStr => ( + + {/* Occupancy row */} + + + {data.dates.map(dateStr => { + const dayData = rateData[dateStr] + const occ = dayData?.occupancy + if (!occ) { + return ( + + ) + } + const bookableRooms = occ.available - occ.maintenance + const roomsLeft = bookableRooms - occ.occupied + const isFull = roomsLeft <= 0 + const occPercent = bookableRooms > 0 + ? Math.round((occ.occupied / bookableRooms) * 100) + : 100 + const isHighOcc = occPercent >= 80 && !isFull + const hasOffline = occ.maintenance > 0 + const getOccStyle = () => { + if (isFull) return styles.occupancyFull + if (isHighOcc) return styles.occupancyHigh + return styles.occupancyAvailable + } + return ( + + ) + })} + + {/* Tariff rows */} + {tariffNames.length === 0 ? ( + + + + ) : ( + tariffNames.map(tariffName => ( + + + {data.dates.map(dateStr => { + const dayData = rateData[dateStr] + const tariff = dayData?.tariffs?.find(t => t.name === tariffName) + const occupancy = dayData?.occupancy + const noRoomsAvailable = occupancy && + (occupancy.available - occupancy.maintenance - occupancy.occupied) <= 0 + + if (!tariff) { + return ( + + ) + } + + const isEffectivelyAvailable = tariff.available || + (tariff.min_stay && tariff.min_stay > 1 && tariff.available_for_min_stay === true) + const minStayBadge = isEffectivelyAvailable && !noRoomsAvailable && tariff.min_stay && tariff.min_stay > 1 ? ( + + {tariff.min_stay} + + ) : null + const getCellStyle = () => { + if (noRoomsAvailable) return styles.cellNoRooms + if (isEffectivelyAvailable) return styles.cellAvailable + return styles.cellUnavailable + } + const getTooltip = () => { + if (noRoomsAvailable) return `${tariffName}: No rooms available` + if (isEffectivelyAvailable) { + const minStayNote = tariff.min_stay && tariff.min_stay > 1 ? ` (Min ${tariff.min_stay} nights)` : '' + return `${tariffName}: ${formatCurrency(tariff.rate)}${minStayNote}` + } + return `${tariffName}: ${tariff.message || 'Not available'}` + } + + return ( + + ) + })} + + )) + )} + + ) + })} + {/* Booking.com section */} + {bookingData && bookingData.has_own_hotel && ( + + + + {data.dates.map(dateStr => ( + + + + {data.dates.map(dateStr => { + const entry = bookingData.dates[dateStr] + const isAvailable = entry?.status === 'available' + const isSoldOut = entry?.status === 'sold_out' + const getCellStyle = () => { + if (!entry) return styles.cellNoData + if (isAvailable && entry.rate) return styles.cellAvailable + if (isSoldOut) return styles.bookingCellSoldOut + return styles.cellNoData + } + return ( + + ) + })} + + + )} + +
Tariff +
+ {formatDayOfWeek(dateStr)} + {formatDateShort(dateStr)} + + {data.date_last_updated?.[dateStr] && ( + + {formatLastUpdated(data.date_last_updated[dateStr])} + + )} +
+
+ {category.category_name} + ({category.room_count} rooms) + + ))} +
+ Occupancy + + - + + + {occ.occupied}/{occ.available} + {hasOffline && ({occ.maintenance})} + +
+ No rate data available for this period +
+ {tariffName} + + - + + + {tariff.rate !== null ? formatCurrency(tariff.rate) : (isEffectivelyAvailable ? 'Y' : 'N')} + {minStayBadge} + +
+ Booking.com + + {' '}{bookingData.latest_scrape ? `Scraped ${formatScrapeAge(bookingData.latest_scrape)}` : 'No scrape data'} + {' · '} + + View details + + + + ))} +
+ Best Available + + {!entry ? '-' + : isAvailable && entry.rate ? formatCurrency(entry.rate) + : isSoldOut ? 'Sold' + : '-'} +
+
+
+ )} +
+ + {/* Issues Panel */} + {summary && summary.unbookableDateCategories > 0 && ( +
+

+ Unbookable Dates ({summary.totalIssues}) +

+

+ Dates with rooms available but no rates bookable +

+
+ {summary.issues.map((issue, idx) => ( +
+ {issue.category} + {issue.date} + {issue.roomsLeft} room{issue.roomsLeft !== 1 ? 's' : ''} available, no rates +
+ ))} + {summary.hasMoreIssues && ( +
+ +{summary.totalIssues - 10} more issues +
+ )} +
+
+ )} + + {/* Legend */} +
+ Legend: + Available + Unavailable + No Rooms + No Data +
+
+ ) +} + +// Styles +const styles: Record = { + container: { + padding: '24px', + maxWidth: '100%', + margin: '0 auto', + }, + header: { + marginBottom: '24px', + }, + headerTop: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'flex-start', + marginBottom: '16px', + flexWrap: 'wrap', + gap: '16px', + }, + title: { + fontSize: '24px', + fontWeight: 700, + color: 'var(--text-dark)', + margin: 0, + }, + subtitle: { + fontSize: '13px', + color: 'var(--text-mid)', + margin: '4px 0 0', + }, + headerActions: { + display: 'flex', + alignItems: 'center', + gap: '16px', + }, + monthSelector: { + display: 'flex', + alignItems: 'center', + gap: '8px', + }, + monthDropdown: { + fontSize: '14px', + fontWeight: 500, + color: 'var(--text-dark)', + padding: '4px 8px', + borderRadius: '6px', + border: '1px solid var(--card-border)', + background: 'var(--card-bg)', + cursor: 'pointer', + minWidth: '160px', + }, + summaryBar: { + display: 'flex', + gap: '24px', + padding: '16px', + background: 'var(--card-bg)', + borderRadius: '10px', + boxShadow: 'var(--shadow-sm)', + flexWrap: 'wrap', + }, + summaryItem: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: '4px', + }, + summaryValue: { + fontSize: '20px', + fontWeight: 700, + color: 'var(--text-dark)', + }, + summaryLabel: { + fontSize: '11px', + color: 'var(--text-mid)', + textTransform: 'uppercase', + }, + content: { + display: 'flex', + flexDirection: 'column', + gap: '24px', + }, + unifiedCard: { + background: 'var(--card-bg)', + borderRadius: '10px', + padding: '16px', + boxShadow: 'var(--shadow-md)', + }, + categoryHeaderRow: { + fontWeight: 600, + fontSize: '14px', + color: 'var(--text-dark)', + background: 'var(--body-bg)', + padding: '8px 16px', + borderTop: '2px solid var(--card-border)', + textAlign: 'left' as const, + whiteSpace: 'nowrap' as const, + }, + categoryHeaderFill: { + background: 'var(--body-bg)', + borderTop: '2px solid var(--card-border)', + padding: 0, + }, + stickyHeader: { + position: 'sticky' as const, + top: 0, + zIndex: 20, + background: 'var(--card-bg)', + }, + roomCount: { + fontSize: '13px', + fontWeight: 400, + color: 'var(--text-mid)', + }, + tableContainer: { + overflowX: 'auto', + maxWidth: '100%', + }, + table: { + width: '100%', + borderCollapse: 'collapse', + fontSize: '13px', + minWidth: '800px', + tableLayout: 'fixed' as const, + }, + th: { + padding: '8px', + borderBottom: '2px solid var(--card-border)', + textAlign: 'center', + fontWeight: 600, + color: 'var(--text-dark)', + whiteSpace: 'nowrap', + background: 'var(--card-bg)', + }, + td: { + padding: '8px', + borderBottom: '1px solid var(--card-border)', + textAlign: 'center', + whiteSpace: 'nowrap', + }, + stickyCol: { + position: 'sticky', + left: 0, + background: 'var(--card-bg)', + zIndex: 10, + textAlign: 'left', + width: '160px', + borderRight: '1px solid var(--card-border)', + overflow: 'hidden', + textOverflow: 'ellipsis', + }, + dateHeader: { + width: '56px', + padding: '4px', + }, + dateHeaderContent: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: '2px', + }, + dayOfWeek: { + fontSize: '11px', + color: 'var(--text-mid)', + }, + dayNum: { + fontSize: '13px', + fontWeight: 600, + }, + lastUpdated: { + fontSize: '9px', + color: 'var(--text-mid)', + opacity: 0.7, + lineHeight: 1, + }, + weekendHeader: { + background: 'var(--body-bg)', + }, + weekendCell: { + borderLeft: '2px solid var(--card-border)', + }, + tariffNameCell: { + fontWeight: 500, + overflow: 'hidden', + textOverflow: 'ellipsis', + }, + tariffCell: { + cursor: 'pointer', + position: 'relative', + transition: 'background 0.1s', + fontSize: '11px', + }, + cellContent: { + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + gap: '2px', + }, + minStayBadge: { + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + width: '14px', + height: '14px', + borderRadius: '50%', + background: '#d97706', + color: '#fff', + fontSize: '9px', + fontWeight: 700, + marginLeft: '2px', + flexShrink: 0, + }, + cellAvailable: { + background: '#dcfce7', + color: 'var(--success)', + }, + cellUnavailable: { + background: '#fee2e2', + color: 'var(--danger)', + textDecoration: 'line-through', + }, + cellNoRooms: { + background: '#e0e0e0', + color: 'var(--text-mid)', + }, + cellNoData: { + background: 'var(--body-bg)', + color: 'var(--text-mid)', + }, + bookingCellSoldOut: { + background: '#fee2e2', + color: 'var(--danger)', + fontWeight: 500, + }, + occupancyLabel: { + fontWeight: 600, + color: 'var(--navy)', + background: '#f0f4ff', + }, + occupancyCell: { + fontSize: '11px', + color: 'var(--text-mid)', + background: 'var(--body-bg)', + }, + occupancyText: { + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + gap: '2px', + }, + maintenanceBadge: { + color: '#d97706', + marginLeft: '2px', + }, + occupancyAvailable: { + background: '#dcfce7', + color: 'var(--success)', + fontWeight: 500, + }, + occupancyHigh: { + background: '#fef3c7', + color: '#d97706', + fontWeight: 500, + }, + occupancyFull: { + background: '#fee2e2', + color: 'var(--danger)', + fontWeight: 500, + }, + loading: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + padding: '48px', + gap: '16px', + color: 'var(--text-mid)', + }, + spinner: { + width: '40px', + height: '40px', + border: '3px solid var(--card-border)', + borderTop: '3px solid var(--navy)', + borderRadius: '50%', + animation: 'spin 1s linear infinite', + }, + error: { + display: 'flex', + alignItems: 'center', + gap: '8px', + padding: '24px', + background: '#fee2e2', + color: 'var(--danger)', + borderRadius: '10px', + }, + errorIcon: { + width: '24px', + height: '24px', + borderRadius: '50%', + background: 'var(--danger)', + color: '#fff', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + fontWeight: 700, + }, + noData: { + textAlign: 'center', + padding: '32px', + color: 'var(--text-mid)', + }, + issuesPanel: { + marginTop: '24px', + background: '#fef3c7', + borderRadius: '10px', + padding: '24px', + }, + issuesTitle: { + fontSize: '14px', + fontWeight: 600, + color: '#d97706', + marginBottom: '4px', + }, + issuesSubtitle: { + fontSize: '13px', + color: 'var(--text-mid)', + marginBottom: '16px', + }, + issuesList: { + display: 'flex', + flexDirection: 'column', + gap: '8px', + }, + issueItem: { + display: 'flex', + gap: '8px', + fontSize: '13px', + flexWrap: 'wrap', + }, + issueCategory: { + fontWeight: 600, + color: 'var(--text-dark)', + }, + issueDate: { + color: 'var(--text-mid)', + }, + issueMessage: { + color: 'var(--danger)', + fontStyle: 'italic', + }, + moreIssues: { + color: 'var(--text-mid)', + fontStyle: 'italic', + marginTop: '8px', + }, + legend: { + display: 'flex', + alignItems: 'center', + gap: '16px', + marginTop: '24px', + padding: '16px', + background: 'var(--card-bg)', + borderRadius: '10px', + fontSize: '13px', + }, + legendTitle: { + fontWeight: 600, + color: 'var(--text-dark)', + }, + legendItem: { + padding: '4px 8px', + borderRadius: '4px', + fontSize: '11px', + }, + scrapeBtn: { + background: 'none', + border: '1px solid var(--card-border)', + borderRadius: '4px', + cursor: 'pointer', + fontSize: '10px', + lineHeight: 1, + padding: '2px 4px', + color: 'var(--text-mid)', + opacity: 0.6, + transition: 'opacity 0.15s', + }, + scrapeBtnActive: { + opacity: 1, + color: 'var(--navy)', + borderColor: 'var(--navy)', + }, +} + +export default Bookability diff --git a/frontend/src/pages/DirectRates.tsx b/frontend/src/pages/DirectRates.tsx new file mode 100644 index 0000000..6652d9d --- /dev/null +++ b/frontend/src/pages/DirectRates.tsx @@ -0,0 +1,520 @@ +import React, { useState } from 'react' +import { useParams, useNavigate } from 'react-router-dom' +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { + Building2, RefreshCw, Plus, Settings, ChevronDown, ChevronRight, + AlertCircle, CheckCircle, Clock, +} from 'lucide-react' +import api from '../api' +import { useAuth } from '../components/AuthGate' +import { can } from '../types' + +const fmtDate = (d: Date) => + `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` + +const TABS = ['overview', 'hotel', 'manage'] as const +type Tab = typeof TABS[number] + +interface DirectHotel { + id: number + name: string + profile_name: string + scrape_enabled: boolean + last_scraped_at: string | null + scraped_dates: number + last_rate_at: string | null +} + +interface DateRow { + stay_date: string + cheapest_rate: number | null + has_availability: boolean + has_min_stay: boolean + scraped_at: string | null +} + +interface RoomRow { + room_id: string + rate_id: string + room_label: string + rate_label: string + availability: number + price_incl: number | null + min_stay_nights: number | null + bench_rate: number | null +} + +const fmt = (v: number | null) => v != null ? `£${v.toFixed(2)}` : '—' +const age = (ts: string | null) => { + if (!ts) return 'Never' + const h = Math.round((Date.now() - new Date(ts).getTime()) / 3600000) + return h < 24 ? `${h}h ago` : `${Math.round(h / 24)}d ago` +} + +export default function DirectRates() { + const { hotelId } = useParams<{ hotelId?: string }>() + const navigate = useNavigate() + const { user } = useAuth() + const canManage = can(user, 'manage_hotels') + + const [tab, setTab] = useState(hotelId ? 'hotel' : 'overview') + const [selectedHotel, setSelectedHotel] = useState(hotelId ? parseInt(hotelId) : null) + const [expandedDate, setExpandedDate] = useState(null) + const [fromDate, setFromDate] = useState(fmtDate(new Date())) + const [toDate, setToDate] = useState(fmtDate(new Date(Date.now() + 89 * 86400000))) + + const { data: hotels, isLoading: hotelsLoading } = useQuery({ + queryKey: ['direct-hotels'], + queryFn: () => api.get('/direct/hotels').then(r => r.data), + }) + + const { data: dates, isLoading: datesLoading } = useQuery<{ dates: DateRow[]; hotel_name: string }>({ + queryKey: ['direct-dates', selectedHotel, fromDate, toDate], + queryFn: () => api.get(`/direct/hotels/${selectedHotel}/dates`, { + params: { from_date: fromDate, to_date: toDate } + }).then(r => r.data), + enabled: !!selectedHotel && tab === 'hotel', + }) + + const { data: roomData } = useQuery<{ rooms: RoomRow[]; bench_price: number | null }>({ + queryKey: ['direct-rooms', selectedHotel, expandedDate], + queryFn: () => api.get(`/direct/hotels/${selectedHotel}/date/${expandedDate}/rooms`).then(r => r.data), + enabled: !!selectedHotel && !!expandedDate, + }) + + const qc = useQueryClient() + const scrapeMutation = useMutation({ + mutationFn: (id: number) => api.post(`/direct/hotels/${id}/scrape`), + onSuccess: () => qc.invalidateQueries({ queryKey: ['direct-hotels'] }), + }) + + const selectHotel = (id: number) => { + setSelectedHotel(id) + setTab('hotel') + setExpandedDate(null) + navigate(`/direct/${id}`) + } + + const presets = [ + { label: '7d', days: 7 }, { label: '14d', days: 14 }, + { label: '30d', days: 30 }, { label: '90d', days: 90 }, + ] + + return ( +
+
+
+
Direct Rates
+
Competitor booking engine rates — scraped directly
+
+
+ +
+ {TABS.map(t => ( + + ))} +
+ + {tab === 'overview' && ( + scrapeMutation.mutate(id)} + scraping={scrapeMutation.isPending} + /> + )} + + {tab === 'hotel' && ( + + )} + + {tab === 'manage' && canManage && ( + qc.invalidateQueries({ queryKey: ['direct-hotels'] })} /> + )} + {tab === 'manage' && !canManage && ( +
You don't have permission to manage competitor hotels.
+ )} +
+ ) +} + +// ─── Overview Tab ───────────────────────────────────────────────────────────── + +function OverviewTab({ hotels, loading, onSelectHotel, onScrape, scraping }: { + hotels: DirectHotel[] + loading: boolean + onSelectHotel: (id: number) => void + onScrape: (id: number) => void + scraping: boolean +}) { + if (loading) return
Loading…
+ if (!hotels.length) return ( +
+ + No competitor hotels configured. Use the Manage tab to add hotels. +
+ ) + + return ( +
+
+ + + + + + + + + + + + + {hotels.map(h => ( + + + + + + + + + ))} + +
HotelEngineDates ScrapedLast ScrapeStatus
+ + {h.profile_name}{h.scraped_dates}{age(h.last_scraped_at)} + + {h.scrape_enabled ? 'Active' : 'Paused'} + + + +
+
+
+ ) +} + +// ─── Hotel Detail Tab ───────────────────────────────────────────────────────── + +function HotelDetailTab({ hotels, selectedHotel, onSelectHotel, dates, hotelName, datesLoading, + fromDate, toDate, setFromDate, setToDate, presets, expandedDate, setExpandedDate, roomData }: any) { + + return ( +
+ {/* Controls */} +
+
+ + +
+
+ + setFromDate(e.target.value)} /> +
+
+ + setToDate(e.target.value)} /> +
+
+ {presets.map((p: any) => ( + + ))} +
+
+ + {!selectedHotel &&
Select a hotel to view rates.
} + + {selectedHotel && datesLoading &&
Loading…
} + + {selectedHotel && !datesLoading && dates.length === 0 && ( +
No rate data for this period. Run a scrape first.
+ )} + + {selectedHotel && !datesLoading && dates.length > 0 && ( +
+
{hotelName} — {dates.length} dates
+
+ + + + + + + + + + + + + {dates.map((d: DateRow) => ( + + setExpandedDate(expandedDate === d.stay_date ? null : d.stay_date)} + style={{ cursor: 'pointer' }} + > + + + + + + + + {expandedDate === d.stay_date && roomData && ( + + + + )} + + ))} + +
DateCheapest RateAvailabilityMin-StayScraped
+ {expandedDate === d.stay_date + ? + : } + {d.stay_date} + {d.cheapest_rate ? `£${Number(d.cheapest_rate).toFixed(2)}` : '—'} + + {d.has_availability + ? + : } + + {d.has_min_stay + ? Min-stay + : null} + {age(d.scraped_at)}
+ +
+
+
+ )} +
+ ) +} + +function RoomBreakdown({ rooms, benchPrice }: { rooms: RoomRow[]; benchPrice: number | null }) { + return ( + + + + + + + + + + + + + {rooms.map((r, i) => ( + + + + + + + + + ))} + +
RoomRate PlanAvailPriceBench RateMin-Stay
{r.room_label}{r.rate_label}{r.availability > 0 ? : }{fmt(r.price_incl)}{fmt(r.bench_rate)}{r.min_stay_nights && r.min_stay_nights > 1 ? {r.min_stay_nights}N : null}
+ ) +} + +// ─── Manage Tab ─────────────────────────────────────────────────────────────── + +function ManageTab({ hotels, onRefresh }: { hotels: DirectHotel[]; onRefresh: () => void }) { + const [showAdd, setShowAdd] = useState(false) + const [detectUrl, setDetectUrl] = useState('') + const [detected, setDetected] = useState(null) + const [detecting, setDetecting] = useState(false) + const [newName, setNewName] = useState('') + const [extraParams, setExtraParams] = useState>({}) + const [profiles, setProfiles] = useState([]) + const qc = useQueryClient() + + const { data: profileList } = useQuery({ + queryKey: ['direct-profiles'], + queryFn: () => api.get('/direct/profiles').then(r => r.data), + }) + + const detectMutation = useMutation({ + mutationFn: (url: string) => api.post('/direct/profiles/detect', { url }), + onSuccess: (res) => setDetected(res.data), + }) + + const createMutation = useMutation({ + mutationFn: (body: any) => api.post('/direct/hotels', body), + onSuccess: () => { qc.invalidateQueries({ queryKey: ['direct-hotels'] }); setShowAdd(false); setDetected(null); setDetectUrl('') }, + }) + + const toggleMutation = useMutation({ + mutationFn: ({ id, enabled }: { id: number; enabled: boolean }) => + api.put(`/direct/hotels/${id}`, { scrape_enabled: enabled }), + onSuccess: () => qc.invalidateQueries({ queryKey: ['direct-hotels'] }), + }) + + const discoveryMutation = useMutation({ + mutationFn: (id: number) => api.post(`/direct/hotels/${id}/discover`), + }) + + const requiredParams = detected + ? (profileList || []).find((p: any) => p.name === detected.profile)?.required_params || [] + : [] + + return ( +
+
+ +
+ + {showAdd && ( +
+
Add Competitor Hotel
+
+
+ +
+ setDetectUrl(e.target.value)} + placeholder="https://booking.eu.guestline.app/..." /> + +
+
+ + {detected && ( + <> +
+ Detected: {detected.profile} engine + {Object.entries(detected).filter(([k]) => k !== 'profile').map(([k, v]) => ( + {k}: {String(v)} + ))} +
+ +
+ + setNewName(e.target.value)} placeholder="e.g. Three Ways House Hotel" style={{ maxWidth: 300 }} /> +
+ + {requiredParams.filter((p: any) => !(p.key in detected)).map((p: any) => ( +
+ + setExtraParams(prev => ({ ...prev, [p.key]: e.target.value }))} + /> +
+ ))} + + + + )} +
+
+ )} + +
+
Configured Competitors
+
+ + + + + + + + + + + {hotels.length === 0 && ( + + )} + {hotels.map(h => ( + + + + + + + ))} + +
NameEngineScrapingActions
No competitors configured yet.
{h.name}{h.profile_name} + + + +
+
+
+
+ ) +} diff --git a/frontend/src/pages/MarketView.tsx b/frontend/src/pages/MarketView.tsx new file mode 100644 index 0000000..1c20701 --- /dev/null +++ b/frontend/src/pages/MarketView.tsx @@ -0,0 +1,1940 @@ +import React, { useState, useMemo, useCallback } from 'react' +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import api from '../api' + +// Format Date as YYYY-MM-DD using local time (avoids UTC/DST shift from toISOString) +const fmtDate = (d: Date) => `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` + +// ============================================ +// TYPES +// ============================================ + +interface ScraperStatus { + enabled: boolean + paused: boolean + pause_until: string | null + backend: string + location_configured: boolean + location_name: string | null + last_scrape: { + batch_id: string + scrape_type: string + started_at: string | null + completed_at: string | null + status: string + hotels_found: number | null + rates_scraped: number | null + error_message: string | null + } | null +} + +interface Hotel { + id: number + booking_com_id: string + name: string + booking_com_url: string | null + star_rating: number | null + review_score: number | null + review_count: number | null + tier: 'own' | 'competitor' | 'market' + display_order: number + notes: string | null + first_seen_at: string | null + last_seen_at: string | null +} + +interface RateMatrixResponse { + from_date: string + to_date: string + dates: string[] + hotels: { + id: number + name: string + tier: string + display_order: number + star_rating: number | null + review_score: number | null + booking_com_url: string | null + }[] + rates: Record> +} + +interface ScheduleInfo { + daily_time: string + today: string + weekday: string + tiers: { + high: { description: string; dates_today: number; range: string | null } + medium: { description: string; dates_today: number; range: string | null } + low: { description: string; dates_today: number; range: string | null } + } + total_dates_today: number +} + +interface QueueStatus { + statuses: Record + retries_pending: number + total_pending: number + total_completed: number + total_failed: number +} + +interface CoverageEntry { + date: string + tier: 'high' | 'medium' | 'low' | 'none' + last_scraped: string | null + next_expected: string | null +} + +interface CoverageResponse { + today: string + coverage: CoverageEntry[] +} + +interface ScrapeHistoryEntry { + batch_id: string + scrape_type: string + started_at: string | null + completed_at: string | null + status: string + dates_queued: number | null + dates_completed: number | null + dates_failed: number | null + hotels_found: number | null + rates_scraped: number | null + error_message: string | null + blocked_at: string | null + resume_after: string | null +} + +// ============================================ +// HELPERS +// ============================================ + +const formatCurrency = (value: number | null): string => { + if (value === null || value === undefined) return '-' + return new Intl.NumberFormat('en-GB', { + style: 'currency', + currency: 'GBP', + minimumFractionDigits: 0, + maximumFractionDigits: 0, + }).format(value) +} + +const formatDateShort = (dateStr: string): string => { + const date = new Date(dateStr + 'T00:00:00') + return date.toLocaleDateString('en-GB', { day: 'numeric' }) +} + +const formatDayOfWeek = (dateStr: string): string => { + const date = new Date(dateStr + 'T00:00:00') + return date.toLocaleDateString('en-GB', { weekday: 'short' }) +} + +const isWeekend = (dateStr: string): boolean => { + const date = new Date(dateStr + 'T00:00:00') + const day = date.getDay() + return day === 0 || day === 6 +} + +const formatDateTime = (iso: string | null): string => { + if (!iso) return '-' + const d = new Date(iso) + return d.toLocaleString('en-GB', { day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit' }) +} + +const formatScrapeAge = (iso: string | null): string => { + if (!iso) return '' + const scraped = new Date(iso) + const now = new Date() + const diffMs = now.getTime() - scraped.getTime() + const diffMins = Math.floor(diffMs / 60000) + if (diffMins < 60) return `${diffMins}m ago` + const diffHours = Math.floor(diffMins / 60) + if (diffHours < 24) return `${diffHours}h ago` + const diffDays = Math.floor(diffHours / 24) + return `${diffDays}d ago` +} + +const tierColor = (tier: string) => { + switch (tier) { + case 'own': return '#2563eb' + case 'competitor': return '#d97706' + case 'market': return '#64748b' + default: return '#64748b' + } +} + +// ============================================ +// INLINE STYLE HELPERS (replacing theme utilities) +// ============================================ + +const mergeStyles = (...s: React.CSSProperties[]): React.CSSProperties => + Object.assign({}, ...s) + +const buttonStyle = (variant: 'primary' | 'secondary' | 'outline', size?: 'small'): React.CSSProperties => { + const base: React.CSSProperties = { + border: 'none', + borderRadius: '6px', + cursor: 'pointer', + fontWeight: 500, + padding: size === 'small' ? '4px 10px' : '8px 16px', + fontSize: size === 'small' ? '13px' : '14px', + lineHeight: 1.4, + transition: 'all 0.15s', + } + if (variant === 'primary') return { ...base, background: 'var(--gold)', color: '#fff' } + if (variant === 'secondary') return { ...base, background: 'var(--navy)', color: '#fff' } + return { ...base, background: 'transparent', color: 'var(--text-dark)', border: '1px solid var(--card-border)' } +} + +const badgeStyle = (variant: 'success' | 'error' | 'warning' | 'info'): React.CSSProperties => { + const map: Record = { + success: { background: '#dcfce7', color: '#16a34a', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + error: { background: '#fee2e2', color: '#dc2626', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + warning: { background: '#fef3c7', color: '#d97706', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + info: { background: '#dbeafe', color: '#2563eb', padding: '2px 8px', borderRadius: '99px', fontSize: '12px', fontWeight: 600 }, + } + return map[variant] || map.info +} + +const inputStyle: React.CSSProperties = { + width: '100%', + padding: '8px 10px', + borderRadius: '6px', + border: '1px solid var(--card-border)', + fontSize: '14px', + color: 'var(--text-dark)', + background: 'var(--card-bg)', + boxSizing: 'border-box', +} + +const inputLabelStyle: React.CSSProperties = { + display: 'block', + fontSize: '12px', + fontWeight: 600, + color: 'var(--text-mid)', + marginBottom: '4px', + textTransform: 'uppercase', + letterSpacing: '0.04em', +} + +// ============================================ +// TABS +// ============================================ + +type TabId = 'matrix' | 'hotels' | 'settings' + +// ============================================ +// STATUS PANEL +// ============================================ + +const StatusPanel: React.FC<{ status: ScraperStatus | undefined, isLoading: boolean }> = ({ status, isLoading }) => { + if (isLoading) return
Loading status...
+ if (!status) return null + + return ( +
+
+ Scraper + + {status.enabled ? 'Enabled' : 'Disabled'} + +
+ {status.paused && ( +
+ Status + + Paused{status.pause_until ? ` until ${formatDateTime(status.pause_until)}` : ''} + +
+ )} +
+ Location + + {status.location_name || 'Not configured'} + +
+
+ Backend + {status.backend} +
+ {status.last_scrape && ( +
+ Last Scrape + + {status.last_scrape.status} + + + {formatDateTime(status.last_scrape.completed_at || status.last_scrape.started_at)} + {status.last_scrape.hotels_found ? ` | ${status.last_scrape.hotels_found} hotels, ${status.last_scrape.rates_scraped} rates` : ''} + +
+ )} +
+ ) +} + +// ============================================ +// SETTINGS TAB +// ============================================ + +const SettingsTab: React.FC = () => { + const queryClient = useQueryClient() + const [locationName, setLocationName] = useState('') + const [pages, setPages] = useState(2) + const [adults, setAdults] = useState(2) + const [scrapeFrom, setScrapeFrom] = useState(() => fmtDate(new Date())) + const [scrapeTo, setScrapeTo] = useState(() => { + const d = new Date() + d.setDate(d.getDate() + 7) + return fmtDate(d) + }) + + const { data: status } = useQuery({ + queryKey: ['scraper-status'], + queryFn: async () => (await api.get('/competitor-rates/status')).data, + }) + + const { data: history } = useQuery({ + queryKey: ['scrape-history'], + queryFn: async () => (await api.get('/competitor-rates/scrape-history?limit=10')).data, + }) + + const { data: scheduleInfo } = useQuery({ + queryKey: ['schedule-info'], + queryFn: async () => (await api.get('/competitor-rates/schedule-info')).data, + }) + + const { data: queueStatus } = useQuery({ + queryKey: ['queue-status'], + queryFn: async () => (await api.get('/competitor-rates/queue-status')).data, + refetchInterval: 30000, + }) + + const { data: coverage } = useQuery({ + queryKey: ['scrape-coverage'], + queryFn: async () => (await api.get('/competitor-rates/scrape-coverage')).data, + staleTime: 60000, + }) + + const setLocationMutation = useMutation({ + mutationFn: async () => { + return (await api.post('/competitor-rates/config/location', { + location_name: locationName, + pages_to_scrape: pages, + adults: adults, + })).data + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['scraper-status'] }) + setLocationName('') + }, + }) + + const enableMutation = useMutation({ + mutationFn: async (enabled: boolean) => { + return (await api.post(`/competitor-rates/config/enable?enabled=${enabled}`)).data + }, + onSuccess: () => queryClient.invalidateQueries({ queryKey: ['scraper-status'] }), + }) + + const unpauseMutation = useMutation({ + mutationFn: async () => (await api.post('/competitor-rates/config/unpause')).data, + onSuccess: () => queryClient.invalidateQueries({ queryKey: ['scraper-status'] }), + }) + + const scrapeMutation = useMutation({ + mutationFn: async () => { + return (await api.post('/competitor-rates/scrape', { + from_date: scrapeFrom, + to_date: scrapeTo, + })).data + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['scraper-status'] }) + queryClient.invalidateQueries({ queryKey: ['scrape-history'] }) + }, + }) + + return ( +
+ {/* Location Configuration */} +
+

Location Configuration

+

+ Set the location for competitor rate scraping. + {status?.location_name && ( + <> Currently: {status.location_name} + )} +

+
+
+ + setLocationName(e.target.value)} + placeholder="e.g. Bowness-on-Windermere" + style={inputStyle} + /> +
+
+ + setPages(parseInt(e.target.value) || 2)} + min={1} + max={5} + style={inputStyle} + /> +
+
+ + setAdults(parseInt(e.target.value) || 2)} + min={1} + max={4} + style={inputStyle} + /> +
+
+ + {setLocationMutation.isError && ( +

+ {(setLocationMutation.error as any)?.response?.data?.detail || 'Failed to set location'} +

+ )} +
+ + {/* Scraper Controls */} +
+

Scraper Controls

+
+ + {status?.paused && ( + + )} +
+
+ + {/* Schedule Info */} +
+

Automatic Schedule

+ {scheduleInfo ? ( +
+

+ Runs daily at {scheduleInfo.daily_time} ({scheduleInfo.weekday}) +

+
+ {Object.entries(scheduleInfo.tiers).map(([key, tier]) => ( +
+
+ {key} + 0 ? 'info' : 'warning')}> + {tier.dates_today} dates + +
+

{tier.description}

+ {tier.range && ( +

{tier.range}

+ )} +
+ ))} +
+

+ Total today: {scheduleInfo.total_dates_today} dates +

+
+ ) : ( +

Loading schedule...

+ )} + + {/* Queue Status */} + {queueStatus && (queueStatus.total_pending > 0 || queueStatus.total_failed > 0) && ( +
+

Queue

+
+ {queueStatus.total_pending > 0 && ( + {queueStatus.total_pending} pending + )} + {queueStatus.retries_pending > 0 && ( + {queueStatus.retries_pending} retries + )} + {queueStatus.total_completed > 0 && ( + {queueStatus.total_completed} done + )} + {queueStatus.total_failed > 0 && ( + {queueStatus.total_failed} failed + )} +
+
+ )} +
+ + {/* Manual Scrape */} +
+

Manual Scrape

+

+ Trigger a one-off scrape for a date range. Runs in background. +

+
+
+ + setScrapeFrom(e.target.value)} + style={inputStyle} + /> +
+
+ + setScrapeTo(e.target.value)} + style={inputStyle} + /> +
+
+ + {!status?.location_configured && ( +

Configure a location first

+ )} + {scrapeMutation.isSuccess && ( +

+ Scrape started! Check status for progress. +

+ )} + {scrapeMutation.isError && ( +

+ {(scrapeMutation.error as any)?.response?.data?.detail || 'Failed to start scrape'} +

+ )} +
+ + {/* Scrape History */} +
+

Scrape History

+ {history && history.length > 0 ? ( +
+ + + + + + + + + + + + + {history.map(entry => ( + + + + + + + + + ))} + +
TypeStartedStatusHotelsRatesError
{entry.scrape_type}{formatDateTime(entry.started_at)} + + {entry.status} + + {entry.hotels_found ?? '-'}{entry.rates_scraped ?? '-'} + {entry.error_message || '-'} +
+
+ ) : ( +

No scrape history yet

+ )} +
+ + {/* Scrape Coverage - 365 day view */} +
+

Scrape Coverage (365 days)

+

+ Each cell is a date. Color shows freshness of data; letter shows priority (H=high, M=medium, L=low). +

+ {coverage ? :

Loading coverage...

} +
+
+ ) +} + +// ============================================ +// COVERAGE GRID +// ============================================ + +const freshnessColor = (lastScraped: string | null): React.CSSProperties => { + if (!lastScraped) return { background: '#e8e8e8', color: '#64748b' } + const hours = (Date.now() - new Date(lastScraped).getTime()) / 3600000 + if (hours < 24) return { background: '#c6efce', color: '#1a7a2e' } // green - fresh + if (hours < 72) return { background: '#fff3cd', color: '#856404' } // yellow - 1-3 days + if (hours < 168) return { background: '#ffe0b2', color: '#e65100' } // orange - 3-7 days + if (hours < 336) return { background: '#f8d7da', color: '#721c24' } // red - 7-14 days + return { background: '#c62828', color: '#ffffff' } // dark red - >14 days +} + +const tierLabel = (tier: string) => { + switch (tier) { + case 'high': return 'H' + case 'medium': return 'M' + case 'low': return 'L' + default: return '-' + } +} + +const CoverageGrid: React.FC<{ coverage: CoverageResponse }> = ({ coverage }) => { + // Group by month + const months = useMemo(() => { + const grouped: Record = {} + for (const entry of coverage.coverage) { + const monthKey = entry.date.substring(0, 7) // YYYY-MM + if (!grouped[monthKey]) grouped[monthKey] = [] + grouped[monthKey].push(entry) + } + return Object.entries(grouped) + }, [coverage.coverage]) + + const formatMonthLabel = (monthKey: string) => { + const [y, m] = monthKey.split('-') + const d = new Date(parseInt(y), parseInt(m) - 1, 1) + return d.toLocaleDateString('en-GB', { month: 'short', year: 'numeric' }) + } + + const formatDateLabel = (dateStr: string) => { + const d = new Date(dateStr + 'T00:00:00') + return d.toLocaleDateString('en-GB', { weekday: 'short', day: 'numeric' }) + } + + const formatAge = (iso: string | null): string => { + if (!iso) return 'Never scraped' + const hours = (Date.now() - new Date(iso).getTime()) / 3600000 + if (hours < 1) return `${Math.floor(hours * 60)}m ago` + if (hours < 24) return `${Math.floor(hours)}h ago` + return `${Math.floor(hours / 24)}d ago` + } + + return ( +
+ {/* Legend */} +
+ {'<'}24h + 1-3d + 3-7d + 7-14d + {'>'} 14d + Never + + H=High M=Medium L=Low priority + +
+ {months.map(([monthKey, entries]) => ( +
+
{formatMonthLabel(monthKey)}
+
+ {entries.map(entry => ( +
+ + {new Date(entry.date + 'T00:00:00').getDate()} + + + {tierLabel(entry.tier)} + +
+ ))} +
+
+ ))} +
+ ) +} + +// ============================================ +// HOTELS TAB +// ============================================ + +const HotelsTab: React.FC = () => { + const queryClient = useQueryClient() + const [tierFilter, setTierFilter] = useState('') + + const { data: hotels, isLoading } = useQuery({ + queryKey: ['competitor-hotels', tierFilter], + queryFn: async () => { + const params = tierFilter ? `?tier=${tierFilter}` : '' + return (await api.get(`/competitor-rates/hotels${params}`)).data + }, + }) + + const tierMutation = useMutation({ + mutationFn: async ({ hotelId, tier }: { hotelId: number, tier: string }) => { + return (await api.put(`/competitor-rates/hotels/${hotelId}/tier`, { tier })).data + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['competitor-hotels'] }) + }, + }) + + const grouped = useMemo(() => { + if (!hotels) return { own: [], competitor: [], market: [] } + return { + own: hotels.filter(h => h.tier === 'own'), + competitor: hotels.filter(h => h.tier === 'competitor'), + market: hotels.filter(h => h.tier === 'market'), + } + }, [hotels]) + + const HotelCard: React.FC<{ hotel: Hotel }> = ({ hotel }) => ( +
+
+
+ {hotel.name} +
+ {hotel.star_rating && {hotel.star_rating} stars} + {hotel.review_score && Score: {hotel.review_score}} + {hotel.review_count && ({hotel.review_count} reviews)} +
+
+
+ +
+
+
+ + ID: {hotel.booking_com_id} + + {hotel.last_seen_at && ( + + Last seen: {formatDateTime(hotel.last_seen_at)} + + )} +
+
+ ) + + if (isLoading) { + return ( +
+
+ Loading hotels... +
+ ) + } + + return ( +
+ {/* Filter */} +
+ Filter: + {['', 'own', 'competitor', 'market'].map(t => ( + + ))} +
+ + {/* Hotels */} + {!hotels || hotels.length === 0 ? ( +
+

No Hotels Discovered

+

+ Run a scrape to discover hotels in your configured location. +

+
+ ) : ( +
+ {/* Own Hotel */} + {grouped.own.length > 0 && ( +
+

+ Your Hotel ({grouped.own.length}) +

+ {grouped.own.map(h => )} +
+ )} + + {/* Competitors */} + {grouped.competitor.length > 0 && ( +
+

+ Competitors ({grouped.competitor.length}) +

+ {grouped.competitor.map(h => )} +
+ )} + + {/* Market */} + {grouped.market.length > 0 && ( +
+

+ Market ({grouped.market.length}) +

+ {grouped.market.map(h => )} +
+ )} +
+ )} +
+ ) +} + +// ============================================ +// RATE MATRIX TAB +// ============================================ + +const MonthSelector: React.FC<{ + value: string + onChange: (value: string) => void +}> = ({ value, onChange }) => { + const handlePrevMonth = () => { + const [year, month] = value.split('-').map(Number) + const date = new Date(year, month - 2, 1) + onChange(`${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`) + } + + const handleNextMonth = () => { + const [year, month] = value.split('-').map(Number) + const date = new Date(year, month, 1) + onChange(`${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`) + } + + const monthOptions = useMemo(() => { + const options: { value: string; label: string }[] = [] + const now = new Date() + for (let i = 0; i < 13; i++) { + const date = new Date(now.getFullYear(), now.getMonth() + i, 1) + const monthValue = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}` + const label = date.toLocaleDateString('en-GB', { month: 'long', year: 'numeric' }) + options.push({ value: monthValue, label }) + } + return options + }, []) + + return ( +
+ + + +
+ ) +} + +const RateMatrixTab: React.FC = () => { + const [selectedMonth, setSelectedMonth] = useState(() => { + const today = new Date() + return `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}` + }) + const [includeMarket, setIncludeMarket] = useState(false) + const [hoveredCell, setHoveredCell] = useState<{ row: number; col: number } | null>(null) + const [scrapingDate, setScrapingDate] = useState(null) + const [rangeMode, setRangeMode] = useState(false) + const [customFrom, setCustomFrom] = useState(fmtDate(new Date())) + const [customTo, setCustomTo] = useState(fmtDate(new Date(Date.now() + 13 * 86400000))) + const [showDirect, setShowDirect] = useState(false) + const queryClient = useQueryClient() + + const dateScrapeM = useMutation({ + mutationFn: async (d: string) => { + setScrapingDate(d) + return (await api.post('/competitor-rates/scrape', { from_date: d, to_date: d })).data + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['competitor-matrix'] }) + queryClient.invalidateQueries({ queryKey: ['scraper-status'] }) + setScrapingDate(null) + }, + onError: () => setScrapingDate(null), + }) + + const onCellEnter = useCallback((row: number, col: number) => { + setHoveredCell({ row, col }) + }, []) + const onCellLeave = useCallback(() => setHoveredCell(null), []) + + const { fromDate, toDate } = useMemo(() => { + if (rangeMode) return { fromDate: customFrom, toDate: customTo } + const [year, month] = selectedMonth.split('-').map(Number) + return { + fromDate: fmtDate(new Date(year, month - 1, 1)), + toDate: fmtDate(new Date(year, month, 0)), + } + }, [selectedMonth, rangeMode, customFrom, customTo]) + + const { data, isLoading, error } = useQuery({ + queryKey: ['competitor-matrix', fromDate, toDate, includeMarket], + queryFn: async () => { + const params = new URLSearchParams({ + from_date: fromDate, + to_date: toDate, + include_market: includeMarket.toString(), + }) + return (await api.get(`/competitor-rates/matrix?${params}`)).data + }, + }) + + const dates = data?.dates || [] + const rates = data?.rates || {} + + // Sort hotels: own first, then competitor, then market + const hotels = useMemo(() => { + const tierPriority: Record = { own: 0, competitor: 1, market: 2 } + return [...(data?.hotels || [])].sort((a, b) => { + const ta = tierPriority[a.tier] ?? 9 + const tb = tierPriority[b.tier] ?? 9 + if (ta !== tb) return ta - tb + return (a.display_order ?? 999) - (b.display_order ?? 999) + }) + }, [data?.hotels]) + + // Compute latest scraped_at per date column across all hotels + const scrapedAtByDate = useMemo(() => { + const result: Record = {} + for (const d of dates) { + let latest: string | null = null + for (const hotel of hotels) { + const rate = (rates[hotel.id] || {})[d] + if (rate?.scraped_at) { + if (!latest || rate.scraped_at > latest) { + latest = rate.scraped_at + } + } + } + result[d] = latest + } + return result + }, [dates, hotels, rates]) + + // Own hotel rate by date (first 'own' tier hotel) + const ownRateByDate = useMemo(() => { + const ownHotel = hotels.find(h => h.tier === 'own') + if (!ownHotel) return {} as Record + const result: Record = {} + for (const d of dates) { + const r = (rates[ownHotel.id] || {})[d] + result[d] = r?.rate_gross ?? null + } + return result + }, [hotels, rates, dates]) + + // Direct rates per competitor hotel per date (cheapest) + const { data: directRatesMap } = useQuery>>({ + queryKey: ['market-direct-rates', fromDate, toDate], + queryFn: async () => { + const compHotels = hotels.filter(h => h.tier === 'competitor' && (h as any).direct_hotel_id) + if (!compHotels.length) return {} + const result: Record> = {} + await Promise.all(compHotels.map(async h => { + const directId = (h as any).direct_hotel_id + try { + const res = await api.get(`/direct/hotels/${directId}/dates`, { + params: { from_date: fromDate, to_date: toDate } + }) + result[h.id] = Object.fromEntries( + (res.data.dates as any[]).map((row: any) => [row.stay_date, row.cheapest_rate]) + ) + } catch { result[h.id] = {} } + })) + return result + }, + enabled: showDirect && hotels.length > 0, + }) + + if (isLoading) { + return ( +
+
+ Loading rate matrix... +
+ ) + } + + if (error) { + return ( +
+ {(error as any)?.response?.data?.detail || 'Failed to load rate matrix'} +
+ ) + } + + return ( +
+ {/* Controls */} +
+ {!rangeMode ? ( + + ) : ( +
+ setCustomFrom(e.target.value)} /> + to + setCustomTo(e.target.value)} /> +
+ )} +
+ {[{ label: '7d', days: 7 }, { label: '14d', days: 14 }, { label: '30d', days: 30 }, { label: '90d', days: 90 }].map(p => ( + + ))} + {rangeMode && ( + + )} +
+ + +
+ + {hotels.length === 0 ? ( +
+

No Rate Data

+

+ Run a scrape and categorize hotels as competitors to see rate comparisons. +

+
+ ) : ( +
+ + + + + {dates.map((d, colIdx) => { + const scrapeAge = formatScrapeAge(scrapedAtByDate[d]) + const isColHovered = hoveredCell?.col === colIdx + return ( + + ) + })} + + + + {hotels.map((hotel, rowIdx) => { + const hotelRates = rates[hotel.id] || {} + const isRowHovered = hoveredCell?.row === rowIdx + return ( + + + + {dates.map((d, colIdx) => { + const rate = hotelRates[d] + const isAvailable = rate?.availability_status === 'available' + const isSoldOut = rate?.availability_status === 'sold_out' + + let cellStyle: React.CSSProperties = styles.matrixCellEmpty + if (rate) { + if (isAvailable && rate.rate_gross) { + cellStyle = styles.matrixCellAvailable + } else if (isSoldOut) { + cellStyle = styles.matrixCellSoldOut + } else { + cellStyle = styles.matrixCellNoRate + } + } + + const tooltip = rate ? [ + rate.room_type, + rate.breakfast_included ? 'Breakfast incl.' : null, + rate.free_cancellation ? 'Free cancel' : null, + rate.rooms_left ? `${rate.rooms_left} left` : null, + ].filter(Boolean).join(' | ') : '' + + // Build booking.com link: strip existing date/guest params, add ours + let bookingUrl: string | null = null + if (hotel.booking_com_url) { + const checkin = d + const co = new Date(d + 'T00:00:00') + co.setDate(co.getDate() + 1) + const checkout = fmtDate(co) + try { + const url = new URL(hotel.booking_com_url) + const stripParams = ['checkin', 'checkout', 'group_adults', 'group_children', 'req_adults', 'req_children', 'no_rooms'] + stripParams.forEach(p => url.searchParams.delete(p)) + url.searchParams.set('checkin', checkin) + url.searchParams.set('checkout', checkout) + url.searchParams.set('group_adults', '2') + bookingUrl = url.toString() + } catch { + // Fallback if URL parsing fails + bookingUrl = hotel.booking_com_url + } + } + + const rawContent = rate ? ( + isAvailable && rate.rate_gross + ? formatCurrency(rate.rate_gross) + : isSoldOut + ? 'Sold' + : '-' + ) : '' + + // Price index badge for competitor rows + let priceIndexBadge: React.ReactNode = null + if (hotel.tier === 'competitor' && rate?.rate_gross && ownRateByDate[d]) { + const idx = Math.round((rate.rate_gross / ownRateByDate[d]!) * 100) + const bg = idx > 105 ? '#dcfce7' : idx < 85 ? '#fee2e2' : idx < 95 ? '#fef3c7' : '#f1f5f9' + const fg = idx > 105 ? '#16a34a' : idx < 85 ? '#dc2626' : idx < 95 ? '#d97706' : '#64748b' + priceIndexBadge = ( + + {idx} + + ) + } + + const cellContent = rawContent + + const isRowH = hoveredCell?.row === rowIdx + const isColH = hoveredCell?.col === colIdx + const isCellH = isRowH && isColH + + return ( + + ) + })} + + {/* Direct rates sub-row */} + {showDirect && hotel.tier === 'competitor' && (directRatesMap?.[hotel.id] != null) && ( + + + {dates.map(d => { + const directRate = directRatesMap?.[hotel.id]?.[d] ?? null + return ( + + ) + })} + + )} + + ) + })} + +
Hotel +
+ {formatDayOfWeek(d)} + {formatDateShort(d)} + {scrapeAge ? ( + {scrapeAge} + ) : null} + +
+
+
+ + {hotel.name} + {hotel.star_rating && ( + {hotel.star_rating}* + )} +
+ 		onCellEnter(rowIdx, colIdx)} + onMouseLeave={onCellLeave} + > + {bookingUrl ? ( + + {cellContent} + + ) : cellContent} + {priceIndexBadge} +
+ Direct + + {directRate ? `£${Number(directRate).toFixed(0)}` : '—'} +
+
+ )} + + {/* Legend */} +
+ Legend: + Available + Sold Out + No Rate + No Data + + Own + Competitor + Market + +
+
+ ) +} + +// ============================================ +// MAIN COMPONENT +// ============================================ + +const CompetitorRates: React.FC = () => { + const [activeTab, setActiveTab] = useState('matrix') + + const { data: status, isLoading: statusLoading } = useQuery({ + queryKey: ['scraper-status'], + queryFn: async () => (await api.get('/competitor-rates/status')).data, + refetchInterval: 30000, + }) + + const tabs: { id: TabId; label: string }[] = [ + { id: 'matrix', label: 'Rate Matrix' }, + { id: 'hotels', label: 'Hotels' }, + { id: 'settings', label: 'Scraper Settings' }, + ] + + return ( +
+ {/* Header */} +
+
+

Competitor Rates

+

+ Compare rates across competitor hotels from Booking.com +

+
+
+ + {/* Status Bar */} + + + {/* Tabs */} +
+ {tabs.map(tab => ( + + ))} +
+ + {/* Tab Content */} +
+ {activeTab === 'matrix' && } + {activeTab === 'hotels' && } + {activeTab === 'settings' && } +
+
+ ) +} + +// ============================================ +// STYLES +// ============================================ + +const styles: Record = { + container: { + padding: '24px', + maxWidth: '100%', + margin: '0 auto', + }, + pageHeader: { + marginBottom: '16px', + }, + title: { + fontSize: '24px', + fontWeight: 700, + color: 'var(--text-dark)', + margin: 0, + }, + subtitle: { + fontSize: '13px', + color: 'var(--text-mid)', + margin: '4px 0 0', + }, + + // Status bar + statusBar: { + display: 'flex', + gap: '24px', + padding: '16px', + background: 'var(--card-bg)', + borderRadius: '10px', + boxShadow: 'var(--shadow-sm)', + marginBottom: '16px', + flexWrap: 'wrap', + alignItems: 'center', + }, + statusItem: { + display: 'flex', + alignItems: 'center', + gap: '8px', + }, + statusLabel: { + fontSize: '11px', + color: 'var(--text-mid)', + textTransform: 'uppercase', + fontWeight: 500, + }, + + // Tabs + tabBar: { + display: 'flex', + gap: '4px', + borderBottom: '2px solid var(--card-border)', + marginBottom: '24px', + }, + tab: { + padding: '8px 24px', + background: 'transparent', + border: 'none', + borderBottom: '2px solid transparent', + cursor: 'pointer', + fontSize: '13px', + fontWeight: 500, + color: 'var(--text-mid)', + marginBottom: '-2px', + transition: 'all 0.2s', + }, + tabActive: { + color: 'var(--navy)', + borderBottomColor: 'var(--navy)', + fontWeight: 600, + }, + tabContent: { + minHeight: '300px', + }, + + // Cards + card: { + background: 'var(--card-bg)', + borderRadius: '10px', + padding: '24px', + boxShadow: 'var(--shadow-md)', + }, + cardTitle: { + fontSize: '16px', + fontWeight: 600, + color: 'var(--text-dark)', + margin: '0 0 4px', + }, + cardDescription: { + fontSize: '13px', + color: 'var(--text-mid)', + margin: '0 0 16px', + }, + + // Settings + settingsGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(350px, 1fr))', + gap: '24px', + }, + formRow: { + display: 'flex', + gap: '16px', + flexWrap: 'wrap', + }, + formGroup: { + flex: 1, + minWidth: '200px', + }, + formGroupSmall: { + width: '80px', + }, + controlRow: { + display: 'flex', + gap: '16px', + flexWrap: 'wrap', + }, + errorText: { + color: 'var(--danger)', + fontSize: '13px', + marginTop: '8px', + }, + hintText: { + color: 'var(--text-mid)', + fontSize: '11px', + marginTop: '8px', + fontStyle: 'italic', + }, + + // Hotels + filterRow: { + display: 'flex', + alignItems: 'center', + gap: '8px', + marginBottom: '24px', + flexWrap: 'wrap', + }, + filterLabel: { + fontSize: '13px', + fontWeight: 500, + color: 'var(--text-mid)', + }, + tierSection: { + marginBottom: '24px', + }, + tierHeader: { + fontSize: '14px', + fontWeight: 600, + margin: '0 0 8px', + }, + hotelCard: { + background: 'var(--card-bg)', + borderRadius: '10px', + padding: '16px', + boxShadow: 'var(--shadow-sm)', + marginBottom: '8px', + }, + hotelHeader: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + gap: '16px', + }, + hotelInfo: { + flex: 1, + }, + hotelName: { + fontSize: '13px', + fontWeight: 600, + color: 'var(--text-dark)', + }, + hotelMeta: { + display: 'flex', + gap: '16px', + fontSize: '11px', + color: 'var(--text-mid)', + marginTop: '4px', + }, + hotelActions: { + display: 'flex', + gap: '8px', + }, + tierSelect: { + padding: '4px 8px', + borderRadius: '6px', + border: '1px solid var(--card-border)', + fontSize: '11px', + cursor: 'pointer', + background: 'var(--card-bg)', + }, + hotelFooter: { + display: 'flex', + justifyContent: 'space-between', + marginTop: '8px', + paddingTop: '8px', + borderTop: '1px solid var(--card-border)', + }, + emptyState: { + textAlign: 'center', + padding: '48px', + background: 'var(--card-bg)', + borderRadius: '10px', + boxShadow: 'var(--shadow-sm)', + }, + + // Rate Matrix + matrixControls: { + display: 'flex', + alignItems: 'center', + gap: '24px', + marginBottom: '24px', + flexWrap: 'wrap', + }, + monthSelector: { + display: 'flex', + alignItems: 'center', + gap: '8px', + }, + monthDropdown: { + fontSize: '14px', + fontWeight: 500, + color: 'var(--text-dark)', + padding: '4px 8px', + borderRadius: '6px', + border: '1px solid var(--card-border)', + background: 'var(--card-bg)', + cursor: 'pointer', + minWidth: '160px', + }, + checkboxLabel: { + display: 'flex', + alignItems: 'center', + gap: '8px', + fontSize: '13px', + color: 'var(--text-mid)', + cursor: 'pointer', + }, + matrixContainer: { + overflowX: 'auto', + background: 'var(--card-bg)', + borderRadius: '10px', + boxShadow: 'var(--shadow-md)', + }, + matrixTable: { + width: '100%', + borderCollapse: 'collapse', + fontSize: '11px', + minWidth: '800px', + }, + matrixTh: { + padding: '8px', + borderBottom: '2px solid var(--card-border)', + textAlign: 'center', + fontWeight: 600, + color: 'var(--text-dark)', + whiteSpace: 'nowrap', + background: 'var(--card-bg)', + fontSize: '11px', + }, + matrixTd: { + padding: '4px 8px', + borderBottom: '1px solid var(--card-border)', + textAlign: 'center', + whiteSpace: 'nowrap', + fontSize: '11px', + }, + stickyCol: { + position: 'sticky', + left: 0, + background: 'var(--card-bg)', + zIndex: 10, + textAlign: 'left', + minWidth: '180px', + maxWidth: '220px', + borderRight: '1px solid var(--card-border)', + }, + hotelNameCell: { + fontWeight: 500, + overflow: 'hidden', + textOverflow: 'ellipsis', + }, + matrixHotelInfo: { + display: 'flex', + alignItems: 'center', + gap: '8px', + }, + tierDot: { + width: '8px', + height: '8px', + borderRadius: '50%', + flexShrink: 0, + display: 'inline-block', + }, + matrixHotelName: { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + }, + matrixStars: { + color: '#d97706', + fontSize: '11px', + flexShrink: 0, + }, + dateHeader: { + minWidth: '50px', + padding: '4px', + }, + dateHeaderContent: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: '2px', + }, + dayOfWeek: { + fontSize: '11px', + color: 'var(--text-mid)', + }, + dayNum: { + fontSize: '13px', + fontWeight: 600, + }, + scrapeAge: { + fontSize: '9px', + color: 'var(--success)', + fontWeight: 400, + opacity: 0.8, + lineHeight: 1, + }, + weekendHeader: { + background: 'var(--body-bg)', + }, + weekendCell: { + borderLeft: '2px solid var(--card-border)', + }, + matrixCellAvailable: { + background: '#dcfce7', + color: 'var(--success)', + fontWeight: 600, + }, + matrixCellSoldOut: { + background: '#fee2e2', + color: 'var(--danger)', + }, + matrixCellNoRate: { + background: '#fef3c7', + color: '#d97706', + }, + matrixCellEmpty: { + background: 'var(--body-bg)', + color: 'var(--text-mid)', + }, + matrixCellLink: { + color: 'inherit', + textDecoration: 'none', + display: 'block', + width: '100%', + height: '100%', + } as React.CSSProperties, + crosshairHighlight: { + boxShadow: 'inset 0 0 0 1px #1a1a2e33', + background: '#1a1a2e08', + }, + crosshairCell: { + boxShadow: 'inset 0 0 0 2px var(--navy)', + }, + crosshairRow: { + boxShadow: 'inset 0 0 0 1px #1a1a2e33', + background: '#1a1a2e08', + }, + crosshairCol: { + boxShadow: 'inset 0 0 0 1px #1a1a2e33', + background: '#1a1a2e08', + }, + scrapeBtn: { + background: 'none', + border: '1px solid var(--card-border)', + borderRadius: '4px', + cursor: 'pointer', + fontSize: '10px', + lineHeight: 1, + padding: '2px 4px', + color: 'var(--text-mid)', + opacity: 0.6, + transition: 'opacity 0.15s', + }, + scrapeBtnActive: { + opacity: 1, + color: 'var(--navy)', + borderColor: 'var(--navy)', + }, + + // Shared + table: { + width: '100%', + borderCollapse: 'collapse', + fontSize: '13px', + }, + th: { + padding: '8px', + borderBottom: '2px solid var(--card-border)', + textAlign: 'left', + fontWeight: 600, + color: 'var(--text-dark)', + whiteSpace: 'nowrap', + fontSize: '11px', + }, + td: { + padding: '8px', + borderBottom: '1px solid var(--card-border)', + fontSize: '13px', + }, + historyTable: { + overflowX: 'auto', + }, + noData: { + textAlign: 'center', + padding: '24px', + color: 'var(--text-mid)', + fontSize: '13px', + }, + legend: { + display: 'flex', + alignItems: 'center', + gap: '16px', + marginTop: '24px', + padding: '16px', + background: 'var(--card-bg)', + borderRadius: '10px', + fontSize: '13px', + flexWrap: 'wrap', + }, + legendTitle: { + fontWeight: 600, + color: 'var(--text-dark)', + }, + legendItem: { + padding: '4px 8px', + borderRadius: '4px', + fontSize: '11px', + }, + loading: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + padding: '48px', + gap: '16px', + color: 'var(--text-mid)', + }, + spinner: { + width: '40px', + height: '40px', + border: '3px solid var(--card-border)', + borderTop: '3px solid var(--navy)', + borderRadius: '50%', + animation: 'spin 1s linear infinite', + }, + errorBox: { + padding: '24px', + background: '#fee2e2', + color: 'var(--danger)', + borderRadius: '10px', + }, + + // Schedule + scheduleGrid: { + display: 'flex', + flexDirection: 'column', + gap: '8px', + }, + scheduleTier: { + padding: '8px', + background: 'var(--body-bg)', + borderRadius: '6px', + }, + scheduleTierHeader: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + }, + scheduleTierName: { + fontSize: '13px', + fontWeight: 600, + color: 'var(--text-dark)', + textTransform: 'capitalize', + }, + scheduleTierDesc: { + fontSize: '11px', + color: 'var(--text-mid)', + margin: '4px 0 0', + }, + scheduleTierRange: { + fontSize: '11px', + color: 'var(--text-mid)', + margin: '2px 0 0', + fontFamily: 'monospace', + }, + queuePanel: { + padding: '8px', + background: 'var(--body-bg)', + borderRadius: '6px', + }, + queueStats: { + display: 'flex', + gap: '8px', + flexWrap: 'wrap', + }, + + // Coverage grid + coverageContainer: { + display: 'flex', + flexDirection: 'column', + gap: '16px', + }, + coverageLegend: { + display: 'flex', + alignItems: 'center', + gap: '8px', + fontSize: '11px', + flexWrap: 'wrap', + }, + coverageLegendItem: { + padding: '2px 8px', + borderRadius: '4px', + fontSize: '11px', + }, + coverageMonth: { + display: 'flex', + alignItems: 'flex-start', + gap: '8px', + }, + coverageMonthLabel: { + fontSize: '11px', + fontWeight: 600, + color: 'var(--text-dark)', + minWidth: '70px', + paddingTop: '3px', + flexShrink: 0, + }, + coverageCells: { + display: 'flex', + flexWrap: 'wrap', + gap: '3px', + }, + coverageCell: { + width: '32px', + height: '28px', + borderRadius: '3px', + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + cursor: 'default', + lineHeight: 1, + border: '1px solid rgba(0,0,0,0.06)', + }, + coverageCellDay: { + fontSize: '9px', + fontWeight: 600, + }, + coverageCellTier: { + fontSize: '7px', + opacity: 0.7, + }, +} + +export default CompetitorRates diff --git a/frontend/src/pages/RateAnalysis.tsx b/frontend/src/pages/RateAnalysis.tsx new file mode 100644 index 0000000..1ebf4bf --- /dev/null +++ b/frontend/src/pages/RateAnalysis.tsx @@ -0,0 +1,362 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import Plot from 'react-plotly.js' +import { TrendingUp, TrendingDown, Minus, AlertTriangle, ChevronDown } from 'lucide-react' +import api from '../api' + +const fmtDate = (d: Date) => + `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` + +interface AnalysisHotel { + hotel_id: number + hotel_name: string + last_scraped: string | null + date_count: number +} + +interface StrategyLabel { + label: string + advance_discount_pct: number + weekend_premium_pct: number + avg_sold_out_rate_pct: number + peak_months: string[] +} + +interface HotelAnalysis { + strategy: StrategyLabel + advance_curve: { days_ahead: number; avg_price: number; sample_count: number }[] + dow_breakdown: { dow: number; dow_name: string; avg_price: number; count: number }[] + sold_out_pattern: { stay_date: string; sold_out_pct: number }[] +} + +interface TimelineEntry { + scraped_at: string + room_id: string + rate_id: string + room_label: string + rate_label: string + price_incl: number | null + availability: number +} + +interface ComparisonRow { + hotel_id: number + hotel_name: string + our_rate: number | null + their_rate: number | null + price_index: number | null + days_checked: number +} + +const DOW = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] +const PLOT_LAYOUT_BASE = { + paper_bgcolor: 'transparent', + plot_bgcolor: 'transparent', + font: { family: 'Inter, system-ui, sans-serif', size: 12, color: '#60748b' }, + margin: { t: 20, r: 16, b: 48, l: 48 }, + showlegend: false, + xaxis: { gridcolor: '#e5e9f0', zeroline: false }, + yaxis: { gridcolor: '#e5e9f0', zeroline: false }, +} + +function strategyIcon(label: string) { + if (label.includes('Discount')) return + if (label.includes('Premium')) return + if (label.includes('Yield')) return + return +} + +function priceIndexClass(idx: number | null) { + if (idx == null) return 'badge badge-neutral' + if (idx > 105) return 'badge badge-success' + if (idx < 85) return 'badge badge-danger' + if (idx < 95) return 'badge badge-warning' + return 'badge badge-neutral' +} + +export default function RateAnalysis() { + const [selectedHotel, setSelectedHotel] = useState(null) + const [timelineDate, setTimelineDate] = useState(fmtDate(new Date(Date.now() + 30 * 86400000))) + const [compFrom, setCompFrom] = useState(fmtDate(new Date())) + const [compTo, setCompTo] = useState(fmtDate(new Date(Date.now() + 29 * 86400000))) + + const { data: hotels } = useQuery({ + queryKey: ['analysis-hotels'], + queryFn: () => api.get('/analysis/hotels').then(r => r.data), + }) + + const { data: analysis, isLoading: analysisLoading } = useQuery({ + queryKey: ['analysis-hotel', selectedHotel], + queryFn: () => api.get(`/analysis/hotel/${selectedHotel}`).then(r => r.data), + enabled: !!selectedHotel, + }) + + const { data: timeline } = useQuery({ + queryKey: ['analysis-timeline', selectedHotel, timelineDate], + queryFn: () => api.get(`/analysis/hotel/${selectedHotel}/timeline`, { + params: { date: timelineDate } + }).then(r => r.data), + enabled: !!selectedHotel, + }) + + const { data: comparison, isLoading: compLoading } = useQuery({ + queryKey: ['analysis-comparison', compFrom, compTo], + queryFn: () => api.get('/analysis/comparison', { + params: { from_date: compFrom, to_date: compTo } + }).then(r => r.data), + enabled: !!(compFrom && compTo), + }) + + const presets = [ + { label: '7d', days: 7 }, { label: '14d', days: 14 }, + { label: '30d', days: 30 }, + ] + + return ( +
+
+
+
Rate Analysis
+
Competitor pricing structure and advance purchase behaviour
+
+
+ + {/* Comparison table — full width, no hotel needed */} +
+
+
+ Market Comparison +
+ {presets.map(p => ( + + ))} + setCompFrom(e.target.value)} /> + to + setCompTo(e.target.value)} /> +
+
+ {compLoading ? ( +
Loading…
+ ) : ( +
+ + + + + + + + + + + + {(comparison || []).length === 0 && ( + + )} + {(comparison || []).map(row => ( + + + + + + + + ))} + +
CompetitorOur Avg RateTheir Avg RatePrice IndexDates Checked
No comparison data available.
+ + {row.our_rate ? `£${Number(row.our_rate).toFixed(2)}` : '—'}{row.their_rate ? `£${Number(row.their_rate).toFixed(2)}` : '—'} + {row.price_index != null ? ( + + {row.price_index.toFixed(0)} + + ) : '—'} + {row.days_checked}
+
+ )} +
+
+ + {/* Hotel selector for deep analysis */} +
+
+
+ + +
+
+
+ + {selectedHotel && analysisLoading && ( +
Loading analysis…
+ )} + + {selectedHotel && !analysisLoading && analysis && ( +
+ {/* Strategy card */} +
+
Pricing Strategy
+
+
+ {strategyIcon(analysis.strategy.label)} + {analysis.strategy.label} +
+ + + + {analysis.strategy.peak_months.length > 0 && ( + + )} +
+
+ +
+ {/* Advance purchase curve */} +
+
Advance Purchase Curve
+
+ {analysis.advance_curve.length > 0 ? ( + p.days_ahead), + y: analysis.advance_curve.map(p => p.avg_price), + line: { color: '#c9a84c', width: 2 }, + marker: { size: 4, color: '#c9a84c' }, + hovertemplate: '%{x} days ahead: £%{y:.2f}', + }]} + layout={{ + ...PLOT_LAYOUT_BASE, + xaxis: { ...PLOT_LAYOUT_BASE.xaxis, title: { text: 'Days ahead', font: { size: 11 } }, autorange: 'reversed' }, + yaxis: { ...PLOT_LAYOUT_BASE.yaxis, title: { text: 'Avg price (£)', font: { size: 11 } }, tickprefix: '£' }, + }} + config={{ displayModeBar: false, responsive: true }} + useResizeHandler + /> + ) : ( +
Not enough data yet.
+ )} +
+
+ + {/* DOW breakdown */} +
+
Day-of-Week Breakdown
+
+ {analysis.dow_breakdown.length > 0 ? ( + d.dow_name), + y: analysis.dow_breakdown.map(d => d.avg_price), + marker: { + color: analysis.dow_breakdown.map(d => + d.dow >= 5 ? '#c9a84c' : '#3b82f6' + ), + }, + hovertemplate: '%{x}: £%{y:.2f}', + }]} + layout={{ + ...PLOT_LAYOUT_BASE, + yaxis: { ...PLOT_LAYOUT_BASE.yaxis, tickprefix: '£' }, + }} + config={{ displayModeBar: false, responsive: true }} + useResizeHandler + /> + ) : ( +
Not enough data yet.
+ )} +
+
+
+ + {/* Rate timeline */} +
+
+ Rate Timeline — How Rates Changed for One Date + setTimelineDate(e.target.value)} /> +
+
+ {(timeline || []).length === 0 ? ( +
No timeline data for this date.
+ ) : ( + + )} +
+
+
+ )} + + {!selectedHotel && ( +
+ Select a competitor above to view their pricing strategy and advance purchase curve. +
+ )} +
+ ) +} + +function StatChip({ label, value, hint }: { label: string; value: string; hint: string }) { + return ( +
+ {label} + {value} + {hint && {hint}} +
+ ) +} + +function buildTimelineTraces(entries: TimelineEntry[]) { + const byRoom: Record = {} + for (const e of entries) { + const key = e.room_label || e.room_id + if (!byRoom[key]) byRoom[key] = [] + byRoom[key].push(e) + } + + const colors = ['#c9a84c', '#3b82f6', '#10b981', '#ef4444', '#8b5cf6', '#f59e0b'] + return Object.entries(byRoom).map(([room, pts], i) => ({ + type: 'scatter' as const, + mode: 'lines+markers' as const, + name: room, + x: pts.map(p => p.scraped_at), + y: pts.map(p => p.price_incl), + line: { color: colors[i % colors.length], width: 2 }, + marker: { size: 5, color: colors[i % colors.length] }, + hovertemplate: `${room}: £%{y:.2f}`, + })) +} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx new file mode 100644 index 0000000..cded341 --- /dev/null +++ b/frontend/src/pages/Settings.tsx @@ -0,0 +1,224 @@ +import { useState } from 'react' +import { useParams, useNavigate } from 'react-router-dom' +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { Save, RefreshCw, Database, Clock } from 'lucide-react' +import api from '../api' + +const TABS = [ + { id: 'newbook', label: 'Newbook Sync' }, + { id: 'system', label: 'System' }, +] + +interface SystemConfig { + [key: string]: string | null +} + +export default function Settings() { + const { tab: tabParam } = useParams<{ tab?: string }>() + const navigate = useNavigate() + const qc = useQueryClient() + const activeTab = tabParam || 'newbook' + + const { data: config, isLoading } = useQuery({ + queryKey: ['system-config'], + queryFn: () => api.get('/competitors/config/system').then(r => r.data), + }) + + const saveMutation = useMutation({ + mutationFn: (payload: { key: string; value: string }) => + api.post('/competitors/config/system', payload), + onSuccess: () => qc.invalidateQueries({ queryKey: ['system-config'] }), + }) + + const syncNow = useMutation({ + mutationFn: () => api.post('/bookability/refresh-all'), + onSuccess: () => qc.invalidateQueries({ queryKey: ['system-config'] }), + }) + + return ( +
+
+
+
Settings
+
Newbook sync and system configuration
+
+
+ +
+ {TABS.map(t => ( + + ))} +
+ + {activeTab === 'newbook' && ( + saveMutation.mutate({ key, value: val })} + onSyncNow={() => syncNow.mutate()} + saving={saveMutation.isPending} + syncing={syncNow.isPending} + /> + )} + + {activeTab === 'system' && ( + + )} +
+ ) +} + +// ─── Newbook Sync Tab ───────────────────────────────────────────────────────── + +interface NewbookTabProps { + config: SystemConfig | undefined + isLoading: boolean + onSave: (key: string, val: string) => void + onSyncNow: () => void + saving: boolean + syncing: boolean +} + +function NewbookTab({ config, isLoading, onSave, onSyncNow, saving, syncing }: NewbookTabProps) { + const [syncTime, setSyncTime] = useState('') + + const syncEnabled = config?.sync_newbook_current_rates_enabled === 'true' + const currentTime = config?.sync_newbook_current_rates_time || '05:20' + + if (isLoading) { + return
Loading…
+ } + + return ( +
+
+
+ + + Newbook Rates Sync + + + {syncEnabled ? 'Enabled' : 'Disabled'} + +
+
+

+ When enabled, the app fetches current tariff rates from the Newbook API daily and + stores them for the Bookability view and rate parity calculations. +

+ +
+ + +
+
+
+ +
+
+ + + Sync Schedule + +
+
+
+ +
+ setSyncTime(e.target.value)} + /> + +
+

+ Current: {currentTime} — Booking.com scraper runs at {config?.booking_scraper_daily_time || '05:30'} +

+
+
+
+
+ ) +} + +// ─── System Tab ─────────────────────────────────────────────────────────────── + +function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; isLoading: boolean }) { + if (isLoading) { + return
Loading…
+ } + + const displayKeys = [ + 'booking_scraper_enabled', + 'booking_scraper_paused', + 'booking_scraper_backend', + 'booking_scraper_daily_time', + 'sync_newbook_current_rates_enabled', + 'sync_newbook_current_rates_time', + ] + + return ( +
+
+
System Configuration
+
+ + + + + + + + + {displayKeys.map(k => ( + + + + + ))} + +
KeyValue
{k} + + {config?.[k] ?? not set} + +
+
+
+ +
+ To configure the Booking.com scraper location and hotel tiers, use the Settings tab inside{' '} + Market View. +
+
+ ) +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts new file mode 100644 index 0000000..a7e16ea --- /dev/null +++ b/frontend/src/types.ts @@ -0,0 +1,114 @@ +export interface User { + email: string + name: string + is_admin: boolean + caps: string[] +} + +export function can(user: User | null, cap: string): boolean { + if (!user) return false + return user.is_admin || user.caps.includes(cap) +} + +export interface Hotel { + id: number + name: string + tier: 'own' | 'competitor' | 'market' + star_rating: number | null + review_score: number | null + booking_com_url: string | null + booking_com_id: string | null + display_order: number + notes: string | null + is_active: boolean + scraped_dates?: number + last_scraped?: string | null +} + +export interface RateMatrixEntry { + hotel_id: number + hotel_name: string + tier: string + rate_date: string + rate_gross: number | null + availability_status: string + rooms_left: number | null + scraped_at: string | null +} + +export interface ScraperStatus { + enabled: boolean + paused: boolean + pause_until: string | null + backend: string + daily_time: string + last_batch: { + batch_id: string + started_at: string + status: string + rates_scraped: number + dates_completed: number + } | null +} + +export interface BookabilityCategory { + category_id: string + category_name: string + room_count: number + display_order: number + dates: BookabilityDate[] +} + +export interface BookabilityDate { + date: string + gross_rate: number | null + tariffs: TariffSummary[] + occupancy_pct: number | null + available_rooms: number | null + is_bookable: boolean +} + +export interface TariffSummary { + tariff_name: string + rate: number | null + min_stay: number | null + advance_max: number | null + is_available: boolean +} + +export interface AdvancePurchaseCurvePoint { + lead_bucket: string + avg_rate: number + sample_count: number +} + +export interface DowAnalysisPoint { + dow: number + dow_label: string + avg_rate: number + date_count: number +} + +export interface StrategySummary { + advance_discount_pct: number | null + weekend_premium_pct: number | null + avg_sold_out_rate_pct: number | null + strategy_label: string +} + +export interface HotelAnalysis { + hotel: Hotel + date_range: { from: string; to: string } + advance_purchase_curve: AdvancePurchaseCurvePoint[] + dow_analysis: DowAnalysisPoint[] + sold_out_pattern: Array<{ dow: number; dow_label: string; total_dates: number; sold_out_dates: number }> + strategy_summary: StrategySummary +} + +export interface RateTimelinePoint { + scraped_at: string + rate_gross: number | null + availability_status: string + rooms_left: number | null + days_out: number +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000..ed7e01b --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' + +export default defineConfig({ + plugins: [react()], + base: '/rates/', +})